From 2cb7a73e252e79f1b9e7122c35a2164206a9501f Mon Sep 17 00:00:00 2001
From: Fred Ji <haifeng.ji@gmail.com>
Date: Thu, 30 Nov 2017 15:29:25 -0800
Subject: [PATCH 01/36] SAMZA-1407 upgrade junit version to 4.12

---
 gradle/dependency-versions.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle
index 4f467ab8d3..e5c93ab769 100644
--- a/gradle/dependency-versions.gradle
+++ b/gradle/dependency-versions.gradle
@@ -31,7 +31,7 @@
   jerseyVersion = "2.22.1"
   jodaTimeVersion = "2.2"
   joptSimpleVersion = "3.2"
-  junitVersion = "4.8.1"
+  junitVersion = "4.12"
   kafkaVersion = "0.10.1.1"
   log4jVersion = "1.2.17"
   metricsVersion = "2.2.0"

From 2490e602e9491fdc5323b29987d91af8b8fe5aaf Mon Sep 17 00:00:00 2001
From: Jacob Maes <jmaes@linkedin.com>
Date: Thu, 30 Nov 2017 17:37:40 -0800
Subject: [PATCH 02/36] SAMZA-1518: Add CPU utilization and file descriptor
 count to JvmMetrics

Author: Jacob Maes <jmaes@linkedin.com>

Reviewers: Jagadish <jvenkatr@linkedin.com>, Prateek Maheshwari <pmaheshw@linkedin.com>

Closes #372 from jmakes/samza-1518
---
 .../versioned/container/metrics-table.html    | 12 ++++++++
 .../org/apache/samza/metrics/JvmMetrics.scala | 28 +++++++++++++++++--
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/docs/learn/documentation/versioned/container/metrics-table.html b/docs/learn/documentation/versioned/container/metrics-table.html
index e504fa3fbd..e2f4e48f9c 100644
--- a/docs/learn/documentation/versioned/container/metrics-table.html
+++ b/docs/learn/documentation/versioned/container/metrics-table.html
@@ -345,6 +345,18 @@ <h1>Samza Metrics Reference</h1>
         <td>threads-terminated</td>
         <td>Current number of terminated threads</td>
     </tr>
+    <tr>
+        <td>process-cpu-usage</td>
+        <td>Current CPU usage of the JVM process as a percentage from 0 to 100. The percentage represents the proportion of executed ticks by the JVM process to the total ticks across all CPUs. A negative number indicates the value was not available from the operating system. For more detail, see the JavaDoc for com.sun.management.OperatingSystemMXBean.</td>
+    </tr>
+    <tr>
+        <td>system-cpu-usage</td>
+        <td>Current CPU usage of the all processes in the whole system as a percentage from 0 to 100. The percentage represents the proportion of executed ticks by all processes to the total ticks across all CPUs. A negative number indicates the value was not available from the operating system. For more detail, see the JavaDoc for com.sun.management.OperatingSystemMXBean.</td>
+    </tr>
+    <tr>
+        <td>open-file-descriptor-count</td>
+        <td>Current number of open file descriptors</td>
+    </tr>
 
     <tr>
         <th colspan="2" class="section" id="system-consumers-metrics">org.apache.samza.system.SystemConsumersMetrics</th>
diff --git a/samza-core/src/main/scala/org/apache/samza/metrics/JvmMetrics.scala b/samza-core/src/main/scala/org/apache/samza/metrics/JvmMetrics.scala
index 6692e35585..f26bd2c24d 100644
--- a/samza-core/src/main/scala/org/apache/samza/metrics/JvmMetrics.scala
+++ b/samza-core/src/main/scala/org/apache/samza/metrics/JvmMetrics.scala
@@ -19,12 +19,14 @@
 
 package org.apache.samza.metrics
 
-import java.lang.management.ManagementFactory
 import scala.collection._
 import scala.collection.JavaConverters._
+import java.lang.management.ManagementFactory
 import java.lang.Thread.State._
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
+
+import com.sun.management.{OperatingSystemMXBean, UnixOperatingSystemMXBean}
 import org.apache.samza.util.Logging
 import org.apache.samza.util.DaemonThreadFactory
 
@@ -40,10 +42,12 @@ object JvmMetrics {
  */
 class JvmMetrics(val registry: MetricsRegistry) extends MetricsHelper with Runnable with Logging {
   final val M = 1024 * 1024.0F
+  final val PCT = 100.0
 
   val memoryMXBean = ManagementFactory.getMemoryMXBean()
   val gcBeans = ManagementFactory.getGarbageCollectorMXBeans()
   val threadMXBean = ManagementFactory.getThreadMXBean()
+  val osMXBean = ManagementFactory.getOperatingSystemMXBean()
   var gcBeanCounters = Map[String, (Counter, Counter)]()
   val executor = Executors.newScheduledThreadPool(1, new DaemonThreadFactory(JvmMetrics.JVM_METRICS_THREAD_NAME_PREFIX))
 
@@ -63,6 +67,11 @@ class JvmMetrics(val registry: MetricsRegistry) extends MetricsHelper with Runna
   val cGcCount = newCounter("gc-count")
   val cGcTimeMillis = newCounter("gc-time-millis")
 
+  // Conditional metrics. Only emitted if the Operating System supports it.
+  val gProcessCpuUsage = if (osMXBean.isInstanceOf[OperatingSystemMXBean]) newGauge("process-cpu-usage", 0.0) else null
+  val gSystemCpuUsage = if (osMXBean.isInstanceOf[OperatingSystemMXBean]) newGauge("system-cpu-usage", 0.0) else null
+  val gOpenFileDescriptorCount = if (osMXBean.isInstanceOf[UnixOperatingSystemMXBean]) newGauge("open-file-descriptor-count", 0.0) else null
+
   def start {
     executor.scheduleWithFixedDelay(this, 0, 5, TimeUnit.SECONDS)
   }
@@ -73,11 +82,12 @@ class JvmMetrics(val registry: MetricsRegistry) extends MetricsHelper with Runna
     updateMemoryUsage
     updateGcUsage
     updateThreadUsage
+    updateOperatingSystemMetrics
 
-    debug("updated metrics to: [%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s]" format
+    debug("updated metrics to: [%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s]" format
       (gMemNonHeapUsedM, gMemNonHeapCommittedM, gMemNonHeapMaxM, gMemHeapUsedM, gMemHeapCommittedM,gMemHeapMaxM, gThreadsNew,
         gThreadsRunnable, gThreadsBlocked, gThreadsWaiting, gThreadsTimedWaiting,
-        gThreadsTerminated, cGcCount, cGcTimeMillis))
+        gThreadsTerminated, cGcCount, cGcTimeMillis, gProcessCpuUsage, gSystemCpuUsage, gOpenFileDescriptorCount))
   }
 
   def stop = executor.shutdown
@@ -153,4 +163,16 @@ class JvmMetrics(val registry: MetricsRegistry) extends MetricsHelper with Runna
     gThreadsTimedWaiting.set(threadsTimedWaiting)
     gThreadsTerminated.set(threadsTerminated)
   }
+
+  private def updateOperatingSystemMetrics {
+    if (osMXBean.isInstanceOf[OperatingSystemMXBean]) {
+      val operatingSystemMXBean = osMXBean.asInstanceOf[OperatingSystemMXBean]
+      gProcessCpuUsage.set(operatingSystemMXBean.getProcessCpuLoad * PCT)
+      gSystemCpuUsage.set(operatingSystemMXBean.getSystemCpuLoad * PCT)
+
+      if (osMXBean.isInstanceOf[UnixOperatingSystemMXBean]) {
+        gOpenFileDescriptorCount.set(osMXBean.asInstanceOf[UnixOperatingSystemMXBean].getOpenFileDescriptorCount)
+      }
+    }
+  }
 }

From 1ca9e3268a324f8c1c74798bbb8661d1b7c9722f Mon Sep 17 00:00:00 2001
From: navina <navina@apache.org>
Date: Fri, 1 Dec 2017 08:57:06 -0800
Subject: [PATCH 03/36] SAMZA-1519: Add release notes to website documentation

Adding a versioned page for release/upgrade notes. We can start this process from the next major version release, aka 0.14.0.

Please update this page as and when you add new features/configs/API or deprecate features/configs/API. Basically, anything that can be useful for Samza users trying to upgrade.

Note: `site.version` is not necessarily same as samza release version. For now, I am using it as a placeholder. Hopefully, with the next generation of our website, it will be better defined.

Author: navina <navina@apache.org>

Reviewers: Jacob Maes <jmaes@linkedin.com>

Closes #301 from navina/versioning
---
 docs/_config.yml                              |  2 +-
 docs/_docs/replace-versioned.sh               |  3 +
 .../releases/versioned/release-notes.md       | 75 +++++++++++++++++++
 3 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 docs/startup/releases/versioned/release-notes.md

diff --git a/docs/_config.yml b/docs/_config.yml
index d81ecac8d8..11850aa28f 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -20,7 +20,7 @@ highlighter: pygments
 markdown: redcarpet
 exclude: ['_notes']
 redcarpet:
-  extensions: ['with_toc_data', 'smart']
+  extensions: ['with_toc_data', 'smart', 'strikethrough']
 exclude: [_docs]
 baseurl: http://samza.apache.org
 version: latest
diff --git a/docs/_docs/replace-versioned.sh b/docs/_docs/replace-versioned.sh
index f148781262..c005e6219d 100755
--- a/docs/_docs/replace-versioned.sh
+++ b/docs/_docs/replace-versioned.sh
@@ -37,5 +37,8 @@ mv -f $DIR/_site/learn/documentation/versioned $DIR/_site/learn/documentation/$v
 echo "replaced learn/tutorials/versioned to learn/tutorials/"$version
 mv -f $DIR/_site/learn/tutorials/versioned $DIR/_site/learn/tutorials/$version
 
+echo "replaced learn/releases/versioned to learn/releases/"$version
+mv -f $DIR/_site/startup/releases/versioned $DIR/_site/startup/releases/$version
+
 echo "replaced startup/hello-samza/versioned to startup/hello-samza/"$version
 mv -f $DIR/_site/startup/hello-samza/versioned $DIR/_site/startup/hello-samza/$version
diff --git a/docs/startup/releases/versioned/release-notes.md b/docs/startup/releases/versioned/release-notes.md
new file mode 100644
index 0000000000..460cbda7b3
--- /dev/null
+++ b/docs/startup/releases/versioned/release-notes.md
@@ -0,0 +1,75 @@
+---
+layout: page
+title: Release Notes
+---
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+{% if site.version != "latest" %}
+1. [Download](#download)
+2. [Release Notes for Samza-{{site.version}} version](#release-notes-for-{{site.version}})
+3. [Upgrade Notes](#upgrade-notes)
+
+{% else %}
+1. [Release Notes for Samza-{{site.version}} version](#release-notes)
+2. [Upgrade Notes](#upgrade-notes)
+{% endif %}
+
+{% if site.version != "latest" %}
+## Download
+
+## Source Release
+[samza-sources-{{site.version}}.tgz](http://www.apache.org/dyn/closer.lua/samza/{{site.version}}.*)
+
+
+{% endif %}
+
+
+
+## Release Notes
+<!-- Add notes on new features, modified behavior of existing features, operational/performance improvements, new tools etc -->
+
+<TBD>
+
+## Upgrade Notes
+<!-- Add detailed notes on how someone using an older version of samza (typically, currentVersion - 1) can upgrade to the latest -->
+<!-- Notes typically include config changes, public-api changes, new user guides/tutorials etc -->
+
+### Configuration Changes
+
+<!-- PR 290 -->
+* Introduced a new **mandatory** configuration - `job.coordination.utils.factory`. Read more about it
+[here](../../learn/{{site.version}}/configuration.html). <br />This config is applicable to all Samza
+applications deployed using the `LocalApplicationRunner` (that is, non-yarn deployments).
+
+### API Changes
+
+<!-- PR 292 -->
+* The following APIs in `SystemAdmin` have been deprecated in the previous versions and hence, replaced with newer APIs.
+If you have a custom **System** implementation, then you have to update to the newer APIs.
+  * ~~void createChangelogStream(String streamName, int numOfPartitions);~~ -> ``` boolean createStream(StreamSpec streamSpec); ```
+  * ~~void createCoordinatorStream(String streamName);~~ -> ``` boolean createStream(StreamSpec streamSpec); ```
+  * ~~void validateChangelogStream(String streamName, int numOfPartitions);~~ -> ``` void validateStream(StreamSpec streamSpec) throws StreamValidationException; ```
+
+<!-- PR 292 -->
+* New API has been added to `SystemAdmin` that clear a stream. <br />
+```
+boolean clearStream(StreamSpec streamSpec);
+```
+<br />
+Read more about it in the [API docs]().
+

From 9c9bbc45cfdbf4d14f27c84ab3dc3314169ade7c Mon Sep 17 00:00:00 2001
From: Abhishek Shivanna <abhisheks91@gmail.com>
Date: Fri, 1 Dec 2017 15:23:12 -0800
Subject: [PATCH 04/36] SAMZA-1506: Fix for robust ContainerHeartbeatMonitor
 exception handling.

The Fix includes the following changes:
- Catch all exceptions inside the heartbeat thread and not just
  IOException.
- A time based force kill when the heartbeat is invalid,
  this makes the monitor immune to threads that may keep the
  container stuck in the shutdown sequence. When the timeout
  occurs, a System.exit(1) is called.
- Increasing number of retries for failed heartbeats from 3 to 6.
  This prevents short intermittent network failurs from causing the
  containers to be invalidated.

Author: Abhishek Shivanna <abhisheks91@gmail.com>

Reviewers: Jacob Maes <jmaes@linkedin.com>

Closes #375 from abhishekshivanna/container-heartbeat
---
 .../container/ContainerHeartbeatClient.java   | 20 ++++++++++++-------
 .../container/ContainerHeartbeatMonitor.java  | 11 +++++++++-
 .../samza/runtime/LocalContainerRunner.java   |  9 +++++++--
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatClient.java b/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatClient.java
index f2c2651d68..7273d54f4b 100644
--- a/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatClient.java
+++ b/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatClient.java
@@ -49,7 +49,7 @@
  */
 public class ContainerHeartbeatClient {
   private static final Logger LOG = LoggerFactory.getLogger(ContainerHeartbeatClient.class);
-  private static final int NUM_RETRIES = 3;
+  private static final int NUM_RETRIES = 6;
   private static final int TIMEOUT_MS = 5000;
   private static final int BACKOFF_MULTIPLIER = 2;
   private final String heartbeatEndpoint;
@@ -72,9 +72,10 @@ public ContainerHeartbeatResponse requestHeartbeat() {
       LOG.debug("Container Heartbeat got response {}", reply);
       response = mapper.readValue(reply, ContainerHeartbeatResponse.class);
       return response;
-    } catch (IOException e) {
-      LOG.error("Error in container heart beat protocol. Query url: {} response: {}", heartbeatEndpoint, reply);
+    } catch (Exception e) {
+      LOG.error("Error in container heartbeat to JobCoordinator.", e);
     }
+    LOG.error("Container heartbeat expired");
     response = new ContainerHeartbeatResponse(false);
     return response;
   }
@@ -82,10 +83,11 @@ public ContainerHeartbeatResponse requestHeartbeat() {
   String httpGet(URL url) throws IOException {
     HttpURLConnection conn;
     int delayMillis = 1000;
-
+    BufferedReader br = null;
     for (int currentTry = 0; currentTry < NUM_RETRIES; currentTry++) {
-      conn = Util.getHttpConnection(url, TIMEOUT_MS);
-      try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
+      try {
+        conn = Util.getHttpConnection(url, TIMEOUT_MS);
+        br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
         if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
           throw new IOException(String.format("HTTP error fetching url %s. Returned status code %d", url.toString(),
               conn.getResponseCode()));
@@ -93,9 +95,13 @@ String httpGet(URL url) throws IOException {
           return br.lines().collect(Collectors.joining());
         }
       } catch (Exception e) {
-        LOG.error("Error in heartbeat request", e);
+        LOG.error(String.format("Error in heartbeat request. Retrying [%d/%d].", currentTry + 1, NUM_RETRIES), e);
         sleepUninterruptibly(delayMillis);
         delayMillis = delayMillis * BACKOFF_MULTIPLIER;
+      } finally {
+        if (br != null) {
+          br.close();
+        }
       }
     }
     throw new IOException(String.format("Error fetching url: %s. Tried %d time(s).", url.toString(), NUM_RETRIES));
diff --git a/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatMonitor.java b/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatMonitor.java
index 940e80f02c..64e74501dc 100644
--- a/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatMonitor.java
+++ b/samza-core/src/main/java/org/apache/samza/container/ContainerHeartbeatMonitor.java
@@ -27,10 +27,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+
 public class ContainerHeartbeatMonitor {
   private static final Logger LOG = LoggerFactory.getLogger(ContainerHeartbeatMonitor.class);
   private static final ThreadFactory THREAD_FACTORY = new HeartbeatThreadFactory();
   private static final int SCHEDULE_MS = 60000;
+  private static final int SHUTDOWN_TIMOUT_MS = 120000;
   private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(THREAD_FACTORY);
   private final Runnable onContainerExpired;
   private final ContainerHeartbeatClient containerHeartbeatClient;
@@ -50,6 +52,11 @@ public void start() {
     scheduler.scheduleAtFixedRate(() -> {
         ContainerHeartbeatResponse response = containerHeartbeatClient.requestHeartbeat();
         if (!response.isAlive()) {
+          scheduler.schedule(() -> {
+              // On timeout of container shutting down, force exit.
+              LOG.error("Graceful shutdown timeout expired. Force exiting.");
+              System.exit(1);
+            }, SHUTDOWN_TIMOUT_MS, TimeUnit.MILLISECONDS);
           onContainerExpired.run();
         }
       }, 0, SCHEDULE_MS, TimeUnit.MILLISECONDS);
@@ -69,7 +76,9 @@ private static class HeartbeatThreadFactory implements ThreadFactory {
 
     @Override
     public Thread newThread(Runnable runnable) {
-      return new Thread(runnable, PREFIX + INSTANCE_NUM.getAndIncrement());
+      Thread t = new Thread(runnable, PREFIX + INSTANCE_NUM.getAndIncrement());
+      t.setDaemon(true);
+      return t;
     }
   }
 }
diff --git a/samza-core/src/main/java/org/apache/samza/runtime/LocalContainerRunner.java b/samza-core/src/main/java/org/apache/samza/runtime/LocalContainerRunner.java
index 50c8181daf..6750ccd2af 100644
--- a/samza-core/src/main/java/org/apache/samza/runtime/LocalContainerRunner.java
+++ b/samza-core/src/main/java/org/apache/samza/runtime/LocalContainerRunner.java
@@ -153,8 +153,13 @@ private void startContainerHeartbeatMonitor() {
     if (executionEnvContainerId != null) {
       log.info("Got execution environment container id: {}", executionEnvContainerId);
       containerHeartbeatMonitor = new ContainerHeartbeatMonitor(() -> {
-          container.shutdown();
-          containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
+          try {
+            container.shutdown();
+            containerRunnerException = new SamzaException("Container shutdown due to expired heartbeat");
+          } catch (Exception e) {
+            log.error("Heartbeat monitor failed to shutdown the container gracefully. Exiting process.", e);
+            System.exit(1);
+          }
         }, new ContainerHeartbeatClient(coordinatorUrl, executionEnvContainerId));
       containerHeartbeatMonitor.start();
     } else {

From 32d2af5dd10b2c2c1cf3f6e683967bafd5285cdf Mon Sep 17 00:00:00 2001
From: Prateek Maheshwari <pmaheshw@linkedin.com>
Date: Mon, 4 Dec 2017 14:21:21 -0800
Subject: [PATCH 05/36] Updated Serde related documentation and error messages
 for High Level API

Updated and clarified the documentation and error messages related to Serdes for Input/Output/PartitionBy streams.

Author: Prateek Maheshwari <pmaheshw@linkedin.com>

Reviewers: Jagadish Venkatraman <vjagadish1989@gmail.com>

Closes #376 from prateekm/documentation-cleanup
---
 .../apache/samza/operators/MessageStream.java | 18 +++++++---
 .../apache/samza/operators/StreamGraph.java   | 33 +++++++++++++++----
 .../org/apache/samza/serializers/KVSerde.java |  8 +++--
 .../samza/operators/MessageStreamImpl.java    |  4 +++
 .../samza/operators/StreamGraphImpl.java      |  6 ++--
 .../operators/impl/InputOperatorImpl.java     |  2 +-
 .../operators/impl/OutputOperatorImpl.java    |  2 +-
 .../impl/PartitionByOperatorImpl.java         |  4 ---
 .../operators/spec/InputOperatorSpec.java     | 10 +++---
 .../operators/spec/OutputStreamImpl.java      | 10 +++---
 .../stream/IntermediateMessageStreamImpl.java | 13 ++++++++
 .../operators/TestMessageStreamImpl.java      |  2 ++
 12 files changed, 80 insertions(+), 32 deletions(-)

diff --git a/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java b/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java
index dcce7c8499..acb2f33fc6 100644
--- a/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java
+++ b/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java
@@ -93,6 +93,9 @@ public interface MessageStream<M> {
 
   /**
    * Allows sending messages in this {@link MessageStream} to an {@link OutputStream}.
+   * <p>
+   * When sending messages to an {@code OutputStream<KV<K, V>>}, messages are partitioned using their serialized key.
+   * When sending messages to any other {@code OutputStream<M>}, messages are partitioned using a null partition key.
    *
    * @param outputStream the output stream to send messages to
    */
@@ -185,8 +188,9 @@ static <T> MessageStream<T> mergeAll(Collection<? extends MessageStream<? extend
    * input to the job.
    * <p>
    * Uses the provided {@link KVSerde} for serialization of keys and values. If the provided {@code serde} is null,
-   * uses the default serde provided via {@link StreamGraph#setDefaultSerde}, which must be a KVSerde.
-   * If no default serde has been provided <b>before</b> calling this method, no-op serdes are used for keys and values.
+   * uses the default serde provided via {@link StreamGraph#setDefaultSerde}, which must be a KVSerde. If the default
+   * serde is not a {@link KVSerde}, a runtime exception will be thrown. If no default serde has been provided
+   * <b>before</b> calling this method, a {@code KVSerde<NoOpSerde, NoOpSerde>} is used.
    * <p>
    * The number of partitions for this intermediate stream is determined as follows:
    * If the stream is an eventual input to a {@link #join}, and the number of partitions for the other stream is known,
@@ -200,8 +204,11 @@ static <T> MessageStream<T> mergeAll(Collection<? extends MessageStream<? extend
    * for any state stores and streams created by this operator (the full ID also contains the job name, job id and
    * operator type). If the application logic is changed, this ID must be reused in the new operator to retain
    * state from the previous version, and changed for the new operator to discard the state from the previous version.
+   * <p>
+   * Unlike {@link #sendTo}, messages with a null key are all sent to partition 0.
    *
-   * @param keyExtractor the {@link Function} to extract the message and partition key from the input message
+   * @param keyExtractor the {@link Function} to extract the message and partition key from the input message.
+   *                     Messages with a null key are all sent to partition 0.
    * @param valueExtractor the {@link Function} to extract the value from the input message
    * @param serde the {@link KVSerde} to use for (de)serializing the key and value.
    * @param id the unique id of this operator in this application
@@ -212,9 +219,12 @@ static <T> MessageStream<T> mergeAll(Collection<? extends MessageStream<? extend
   <K, V> MessageStream<KV<K, V>> partitionBy(Function<? super M, ? extends K> keyExtractor,
       Function<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String id);
 
-
   /**
    * Same as calling {@link #partitionBy(Function, Function, KVSerde, String)} with a null KVSerde.
+   * <p>
+   * Uses the default serde provided via {@link StreamGraph#setDefaultSerde}, which must be a KVSerde. If the default
+   * serde is not a {@link KVSerde}, a runtime exception will be thrown. If no default serde has been provided
+   * <b>before</b> calling this method, a {@code KVSerde<NoOpSerde, NoOpSerde>} is used.
    *
    * @param keyExtractor the {@link Function} to extract the message and partition key from the input message
    * @param valueExtractor the {@link Function} to extract the value from the input message
diff --git a/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java b/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java
index 17223b1b60..49306316db 100644
--- a/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java
+++ b/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java
@@ -32,8 +32,10 @@ public interface StreamGraph {
    * Sets the default {@link Serde} to use for (de)serializing messages.
    * <p>.
    * If the default serde is set, it must be set <b>before</b> creating any input or output streams.
-   * If no explicit or default serdes are provided, a NoOpSerde is used for keys and values. This means that any
-   * streams created without explicit or default serdes should be cast to MessageStream&lt;KV&lt;Object, Object&gt;&gt;.
+   * <p>
+   * If no explicit or default serdes are provided, a {@code KVSerde<NoOpSerde, NoOpSerde>} is used. This means that
+   * any streams created without explicit or default serdes should be cast to {@code MessageStream<KV<Object, Object>>}.
+   * <p>
    * Providing an incompatible message type for the input/output streams that use the default serde will result in
    * {@link ClassCastException}s at runtime.
    *
@@ -44,6 +46,13 @@ public interface StreamGraph {
   /**
    * Gets the input {@link MessageStream} corresponding to the {@code streamId}.
    * <p>
+   * An input {@code MessageStream<KV<K, V>}, which can be obtained by calling this method with a {@code KVSerde<K, V>},
+   * can receive messages of type {@code KV<K, V>}. An input {@code MessageStream<M>} with any other {@code Serde<M>}
+   * can receive messages of type M - the key in the incoming message is ignored.
+   * <p>
+   * A {@code KVSerde<NoOpSerde, NoOpSerde>} or {@code NoOpSerde} may be used if the {@code SystemConsumer}
+   * deserializes the incoming messages itself, and no further deserialization is required from the framework.
+   * <p>
    * Multiple invocations of this method with the same {@code streamId} will throw an {@link IllegalStateException}.
    *
    * @param streamId the unique ID for the stream
@@ -58,9 +67,10 @@ public interface StreamGraph {
    * Same as {@link #getInputStream(String, Serde)}, but uses the default {@link Serde} provided via
    * {@link #setDefaultSerde(Serde)} for deserializing input messages.
    * <p>
-   * If no default serde has been provided <b>before</b> calling this method, a no-op serde is used.
-   * Providing a message type {@code M} that is incompatible with the default Serde will result in
+   * If no default serde has been provided <b>before</b> calling this method, a {@code KVSerde<NoOpSerde, NoOpSerde>}
+   * is used. Providing a message type {@code M} that is incompatible with the default Serde will result in
    * {@link ClassCastException}s at runtime.
+   * <p>
    * Multiple invocations of this method with the same {@code streamId} will throw an {@link IllegalStateException}.
    *
    * @param streamId the unique ID for the stream
@@ -73,6 +83,16 @@ public interface StreamGraph {
   /**
    * Gets the {@link OutputStream} corresponding to the {@code streamId}.
    * <p>
+   * An {@code OutputStream<KV<K, V>>}, which can be obtained by calling this method with a {@code KVSerde<K, V>},
+   * can send messages of type {@code KV<K, V>}. An {@code OutputStream<M>} with any other {@code Serde<M>} can
+   * send messages of type M without a key.
+   * <p>
+   * A {@code KVSerde<NoOpSerde, NoOpSerde>} or {@code NoOpSerde} may be used if the {@code SystemProducer}
+   * serializes the outgoing messages itself, and no prior serialization is required from the framework.
+   * <p>
+   * When sending messages to an {@code OutputStream<KV<K, V>>}, messages are partitioned using their serialized key.
+   * When sending messages to any other {@code OutputStream<M>}, messages are partitioned using a null partition key.
+   * <p>
    * Multiple invocations of this method with the same {@code streamId} will throw an {@link IllegalStateException}.
    *
    * @param streamId the unique ID for the stream
@@ -87,9 +107,10 @@ public interface StreamGraph {
    * Same as {@link #getOutputStream(String, Serde)}, but uses the default {@link Serde} provided via
    * {@link #setDefaultSerde(Serde)} for serializing output messages.
    * <p>
-   * If no default serde has been provided <b>before</b> calling this method, a no-op serde is used.
-   * Providing a message type {@code M} that is incompatible with the default Serde will result in
+   * If no default serde has been provided <b>before</b> calling this method, a {@code KVSerde<NoOpSerde, NoOpSerde>}
+   * is used. Providing a message type {@code M} that is incompatible with the default Serde will result in
    * {@link ClassCastException}s at runtime.
+   * <p>
    * Multiple invocations of this method with the same {@code streamId} will throw an {@link IllegalStateException}.
    *
    * @param streamId the unique ID for the stream
diff --git a/samza-api/src/main/java/org/apache/samza/serializers/KVSerde.java b/samza-api/src/main/java/org/apache/samza/serializers/KVSerde.java
index 10f224ba6f..42a527d7a1 100644
--- a/samza-api/src/main/java/org/apache/samza/serializers/KVSerde.java
+++ b/samza-api/src/main/java/org/apache/samza/serializers/KVSerde.java
@@ -25,8 +25,12 @@
 
 
 /**
- * A marker serde class to indicate that messages are keyed and should be deserialized as K-V pairs. This class is
- * intended for use cases where a single Serde parameter or configuration is required.
+ * A serializer that serves one of the following two purposes:
+ * <ol>
+ *   <li>When used for input/output streams, serves as an indication to the framework that messages are keyed and
+ *   should be deserialized as K-V pairs.</li>
+ *   <li>When used for a key or value of type KV, serializes the key and value pair using the provided serdes.</li>
+ * </ol>
  *
  * @param <K> type of the key in the message
  * @param <V> type of the value in the message
diff --git a/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java b/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java
index e6e711c01d..3f4e40d45e 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java
@@ -152,6 +152,10 @@ public <K, V> MessageStream<KV<K, V>> partitionBy(Function<? super M, ? extends
       Function<? super M, ? extends V> valueExtractor, KVSerde<K, V> serde, String userDefinedId) {
     String opId = this.graph.getNextOpId(OpCode.PARTITION_BY, userDefinedId);
     IntermediateMessageStreamImpl<KV<K, V>> intermediateStream = this.graph.getIntermediateStream(opId, serde);
+    if (!intermediateStream.isKeyed()) {
+      // this can only happen when the default serde partitionBy variant is being used
+      throw new SamzaException("partitionBy can not be used with a default serde that is not a KVSerde.");
+    }
     PartitionByOperatorSpec<M, K, V> partitionByOperatorSpec =
         OperatorSpecs.createPartitionByOperatorSpec(
             intermediateStream.getOutputStream(), keyExtractor, valueExtractor, opId);
diff --git a/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java b/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java
index 5323c32342..d014cb9419 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java
@@ -108,8 +108,7 @@ public <M> MessageStream<M> getInputStream(String streamId, Serde<M> serde) {
     InputOperatorSpec inputOperatorSpec =
         OperatorSpecs.createInputOperatorSpec(streamSpec, kvSerdes.getKey(), kvSerdes.getValue(),
             isKeyed, this.getNextOpId(OpCode.INPUT, null));
-    inputOperators.put(streamSpec,
-        inputOperatorSpec);
+    inputOperators.put(streamSpec, inputOperatorSpec);
     return new MessageStreamImpl<>(this, inputOperators.get(streamSpec));
   }
 
@@ -137,8 +136,7 @@ public <M> OutputStream<M> getOutputStream(String streamId, Serde<M> serde) {
     }
 
     boolean isKeyed = serde instanceof KVSerde;
-    outputStreams.put(streamSpec,
-        new OutputStreamImpl<>(streamSpec, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed));
+    outputStreams.put(streamSpec, new OutputStreamImpl<>(streamSpec, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed));
     return outputStreams.get(streamSpec);
   }
 
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/InputOperatorImpl.java b/samza-core/src/main/java/org/apache/samza/operators/impl/InputOperatorImpl.java
index 9cc5370325..ef3c322d5e 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/impl/InputOperatorImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/InputOperatorImpl.java
@@ -50,7 +50,7 @@ protected void handleInit(Config config, TaskContext context) {
 
   @Override
   public Collection<Object> handleMessage(KV<K, V> pair, MessageCollector collector, TaskCoordinator coordinator) {
-    Object message = this.inputOpSpec.isKeyedInput() ? pair : pair.getValue();
+    Object message = this.inputOpSpec.isKeyed() ? pair : pair.getValue();
     return Collections.singletonList(message);
   }
 
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/OutputOperatorImpl.java b/samza-core/src/main/java/org/apache/samza/operators/impl/OutputOperatorImpl.java
index 7b7e49ce87..27bef877fd 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/impl/OutputOperatorImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/OutputOperatorImpl.java
@@ -57,7 +57,7 @@ protected void handleInit(Config config, TaskContext context) {
   public Collection<Void> handleMessage(M message, MessageCollector collector,
       TaskCoordinator coordinator) {
     Object key, value;
-    if (outputStream.isKeyedOutput()) {
+    if (outputStream.isKeyed()) {
       key = ((KV) message).getKey();
       value = ((KV) message).getValue();
     } else {
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/PartitionByOperatorImpl.java b/samza-core/src/main/java/org/apache/samza/operators/impl/PartitionByOperatorImpl.java
index b3fb4b2a82..9fc1e7c65f 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/impl/PartitionByOperatorImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/PartitionByOperatorImpl.java
@@ -18,7 +18,6 @@
  */
 package org.apache.samza.operators.impl;
 
-import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
 import org.apache.samza.container.TaskContextImpl;
 import org.apache.samza.operators.KV;
@@ -55,9 +54,6 @@ class PartitionByOperatorImpl<M, K, V> extends OperatorImpl<M, Void> {
   PartitionByOperatorImpl(PartitionByOperatorSpec<M, K, V> partitionByOpSpec, Config config, TaskContext context) {
     this.partitionByOpSpec = partitionByOpSpec;
     OutputStreamImpl<KV<K, V>> outputStream = partitionByOpSpec.getOutputStream();
-    if (!outputStream.isKeyedOutput()) {
-      throw new SamzaException("Output stream for repartitioning must be a keyed stream.");
-    }
     this.systemStream = new SystemStream(
         outputStream.getStreamSpec().getSystemName(),
         outputStream.getStreamSpec().getPhysicalName());
diff --git a/samza-core/src/main/java/org/apache/samza/operators/spec/InputOperatorSpec.java b/samza-core/src/main/java/org/apache/samza/operators/spec/InputOperatorSpec.java
index 4c0687a3b3..2ad0597c0f 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/spec/InputOperatorSpec.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/spec/InputOperatorSpec.java
@@ -35,15 +35,15 @@ public class InputOperatorSpec<K, V> extends OperatorSpec<KV<K, V>, Object> { //
   private final StreamSpec streamSpec;
   private final Serde<K> keySerde;
   private final Serde<V> valueSerde;
-  private final boolean isKeyedInput;
+  private final boolean isKeyed;
 
   public InputOperatorSpec(StreamSpec streamSpec,
-      Serde<K> keySerde, Serde<V> valueSerde, boolean isKeyedInput, String opId) {
+      Serde<K> keySerde, Serde<V> valueSerde, boolean isKeyed, String opId) {
     super(OpCode.INPUT, opId);
     this.streamSpec = streamSpec;
     this.keySerde = keySerde;
     this.valueSerde = valueSerde;
-    this.isKeyedInput = isKeyedInput;
+    this.isKeyed = isKeyed;
   }
 
   public StreamSpec getStreamSpec() {
@@ -58,8 +58,8 @@ public Serde<V> getValueSerde() {
     return valueSerde;
   }
 
-  public boolean isKeyedInput() {
-    return isKeyedInput;
+  public boolean isKeyed() {
+    return isKeyed;
   }
 
   @Override
diff --git a/samza-core/src/main/java/org/apache/samza/operators/spec/OutputStreamImpl.java b/samza-core/src/main/java/org/apache/samza/operators/spec/OutputStreamImpl.java
index a793e0cb46..e439c4e178 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/spec/OutputStreamImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/spec/OutputStreamImpl.java
@@ -28,14 +28,14 @@ public class OutputStreamImpl<M> implements OutputStream<M> {
   private final StreamSpec streamSpec;
   private final Serde keySerde;
   private final Serde valueSerde;
-  private final boolean isKeyedOutput;
+  private final boolean isKeyed;
 
   public OutputStreamImpl(StreamSpec streamSpec,
-      Serde keySerde, Serde valueSerde, boolean isKeyedOutput) {
+      Serde keySerde, Serde valueSerde, boolean isKeyed) {
     this.streamSpec = streamSpec;
     this.keySerde = keySerde;
     this.valueSerde = valueSerde;
-    this.isKeyedOutput = isKeyedOutput;
+    this.isKeyed = isKeyed;
   }
 
   public StreamSpec getStreamSpec() {
@@ -50,7 +50,7 @@ public Serde getValueSerde() {
     return valueSerde;
   }
 
-  public boolean isKeyedOutput() {
-    return isKeyedOutput;
+  public boolean isKeyed() {
+    return isKeyed;
   }
 }
diff --git a/samza-core/src/main/java/org/apache/samza/operators/stream/IntermediateMessageStreamImpl.java b/samza-core/src/main/java/org/apache/samza/operators/stream/IntermediateMessageStreamImpl.java
index 279cdd4844..5eeca99e6d 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/stream/IntermediateMessageStreamImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/stream/IntermediateMessageStreamImpl.java
@@ -25,6 +25,8 @@
 import org.apache.samza.operators.spec.OperatorSpec;
 import org.apache.samza.operators.spec.OutputStreamImpl;
 import org.apache.samza.system.StreamSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * An intermediate stream is both an input and an output stream (e.g. a repartitioned stream).
@@ -39,12 +41,19 @@
  */
 public class IntermediateMessageStreamImpl<M> extends MessageStreamImpl<M> implements OutputStream<M> {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(IntermediateMessageStreamImpl.class);
   private final OutputStreamImpl<M> outputStream;
+  private final boolean isKeyed;
 
   public IntermediateMessageStreamImpl(StreamGraphImpl graph, InputOperatorSpec<?, M> inputOperatorSpec,
       OutputStreamImpl<M> outputStream) {
     super(graph, (OperatorSpec<?, M>) inputOperatorSpec);
     this.outputStream = outputStream;
+    if (inputOperatorSpec.isKeyed() != outputStream.isKeyed()) {
+      LOGGER.error("Input and output streams for intermediate stream {} aren't keyed consistently. Input: {}, Output: {}",
+          new Object[]{inputOperatorSpec.getStreamSpec().getId(), inputOperatorSpec.isKeyed(), outputStream.isKeyed()});
+    }
+    this.isKeyed = inputOperatorSpec.isKeyed() && outputStream.isKeyed();
   }
 
   public StreamSpec getStreamSpec() {
@@ -54,4 +63,8 @@ public StreamSpec getStreamSpec() {
   public OutputStreamImpl<M> getOutputStream() {
     return this.outputStream;
   }
+
+  public boolean isKeyed() {
+    return isKeyed;
+  }
 }
diff --git a/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java b/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java
index 5b3c3a0109..d97d4945fe 100644
--- a/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java
+++ b/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java
@@ -207,6 +207,7 @@ public void testRepartition() {
         .thenReturn(mockIntermediateStream);
     when(mockIntermediateStream.getOutputStream())
         .thenReturn(mockOutputStreamImpl);
+    when(mockIntermediateStream.isKeyed()).thenReturn(true);
 
     MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);
     Function mockKeyFunction = mock(Function.class);
@@ -236,6 +237,7 @@ public void testRepartitionWithoutSerde() {
         .thenReturn(mockIntermediateStream);
     when(mockIntermediateStream.getOutputStream())
         .thenReturn(mockOutputStreamImpl);
+    when(mockIntermediateStream.isKeyed()).thenReturn(true);
 
     MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);
     Function mockKeyFunction = mock(Function.class);

From e3efdf5c872cda681d5d012046524b9992ffda91 Mon Sep 17 00:00:00 2001
From: Bharath Kumarasubramanian <bkumaras@linkedin.com>
Date: Mon, 4 Dec 2017 17:11:03 -0800
Subject: [PATCH 06/36] Fix compile errors with scala 2.12 and update release
 notes to use check all

Author: Bharath Kumarasubramanian <bkumaras@linkedin.com>

Reviewers: Xinyu Liu <xinyuliu.us@gmail.com>

Closes #378 from bharathkk/master
---
 RELEASE.md                                           |  4 ++--
 .../samza/storage/kv/RocksDbKeyValueStore.scala      | 12 ++++++------
 .../apache/samza/storage/kv/AccessLoggedStore.scala  |  4 ++--
 .../samza/storage/kv/KeyValueStorageEngine.scala     |  4 ++--
 .../org/apache/samza/storage/kv/LoggedStore.scala    |  4 ++--
 .../samza/storage/kv/NullSafeKeyValueStore.scala     |  4 ++--
 .../samza/storage/kv/SerializedKeyValueStore.scala   |  4 ++--
 7 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/RELEASE.md b/RELEASE.md
index 2f38c8d82a..cb83594a49 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -30,9 +30,9 @@ And before you proceed, do the following steps:
    * change the samza-test versions in samza-test/src/main/config/join/README to $VERSION w/o the suffix '-SNAPSHOT'.
    * push the changes to the $VERSION branch
 
-Validate that all Samza source files have proper license information in their header.
+Validate Samza using all our supported build matrix.
 
-    ./gradlew check
+    ./bin/check-all.sh
 
 To release to a local Maven repository:
 
diff --git a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala
index 2ae4bb0113..6aad45f573 100644
--- a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala
+++ b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala
@@ -122,7 +122,7 @@ class RocksDbKeyValueStore(
     found
   }
 
-  def getAll(keys: java.util.List[Array[Byte]]): java.util.Map[Array[Byte], Array[Byte]] = ifOpen {
+  override def getAll(keys: java.util.List[Array[Byte]]): java.util.Map[Array[Byte], Array[Byte]] = ifOpen {
     metrics.getAlls.inc
     require(keys != null, "Null keys not allowed.")
     val map = db.multiGet(keys)
@@ -251,10 +251,10 @@ class RocksDbKeyValueStore(
       new Entry(key, value)
     }
 
-    // By virtue of how RocksdbIterator is implemented, the implementation of 
-    // our iterator is slightly different from standard java iterator next will 
-    // always point to the current element, when next is called, we return the 
-    // current element we are pointing to and advance the iterator to the next 
+    // By virtue of how RocksdbIterator is implemented, the implementation of
+    // our iterator is slightly different from standard java iterator next will
+    // always point to the current element, when next is called, we return the
+    // current element we are pointing to and advance the iterator to the next
     // location (The new location may or may not be valid - this will surface
     // when the next next() call is made, the isValid will fail)
     override def next(): Entry[Array[Byte], Array[Byte]] = ifOpen {
@@ -280,7 +280,7 @@ class RocksDbKeyValueStore(
   }
 
   class RocksDbRangeIterator(iter: RocksIterator, from: Array[Byte], to: Array[Byte]) extends RocksDbIterator(iter) {
-    // RocksDB's JNI interface does not expose getters/setters that allow the 
+    // RocksDB's JNI interface does not expose getters/setters that allow the
     // comparator to be pluggable, and the default is lexicographic, so it's
     // safe to just force lexicographic comparator here for now.
     val comparator: LexicographicComparator = lexicographic
diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/AccessLoggedStore.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/AccessLoggedStore.scala
index c21c9a6527..879a144850 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/AccessLoggedStore.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/AccessLoggedStore.scala
@@ -56,7 +56,7 @@ class AccessLoggedStore[K, V](
     logAccess(DBOperation.READ, list, store.get(key))
   }
 
-  def getAll(keys: util.List[K]): util.Map[K, V] = {
+  override def getAll(keys: util.List[K]): util.Map[K, V] = {
     logAccess(DBOperation.READ, serializeKeys(keys), store.getAll(keys))
   }
 
@@ -76,7 +76,7 @@ class AccessLoggedStore[K, V](
     logAccess(DBOperation.DELETE, list, store.delete(key))
   }
 
-  def deleteAll(keys: util.List[K]): Unit = {
+  override def deleteAll(keys: util.List[K]): Unit = {
     logAccess(DBOperation.DELETE, serializeKeys(keys), store.deleteAll(keys))
   }
 
diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala
index 8385c4a8cc..646b60608c 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala
@@ -51,7 +51,7 @@ class KeyValueStorageEngine[K, V](
     }
   }
 
-  def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
+  override def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
     metrics.gets.inc(keys.size)
     wrapperStore.getAll(keys)
   }
@@ -75,7 +75,7 @@ class KeyValueStorageEngine[K, V](
     }
   }
 
-  def deleteAll(keys: java.util.List[K]) = {
+  override def deleteAll(keys: java.util.List[K]) = {
     metrics.deletes.inc(keys.size)
     wrapperStore.deleteAll(keys)
   }
diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/LoggedStore.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/LoggedStore.scala
index 33ff41b231..e0c7a31628 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/LoggedStore.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/LoggedStore.scala
@@ -41,7 +41,7 @@ class LoggedStore[K, V](
     store.get(key)
   }
 
-  def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
+  override def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
     metrics.gets.inc(keys.size)
     store.getAll(keys)
   }
@@ -90,7 +90,7 @@ class LoggedStore[K, V](
   /**
    * Perform the local deletes and log them out to the changelog
    */
-  def deleteAll(keys: java.util.List[K]) = {
+  override def deleteAll(keys: java.util.List[K]) = {
     metrics.deletes.inc(keys.size)
     val keysIterator = keys.iterator
     while (keysIterator.hasNext) {
diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/NullSafeKeyValueStore.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/NullSafeKeyValueStore.scala
index 9e67fc8b6b..3cc35d3688 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/NullSafeKeyValueStore.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/NullSafeKeyValueStore.scala
@@ -37,7 +37,7 @@ class NullSafeKeyValueStore[K, V](store: KeyValueStore[K, V]) extends KeyValueSt
     store.get(key)
   }
 
-  def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
+  override def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
     notNull(keys, NullKeysErrorMessage)
     keys.asScala.foreach(key => notNull(key, NullKeyErrorMessage))
     store.getAll(keys)
@@ -62,7 +62,7 @@ class NullSafeKeyValueStore[K, V](store: KeyValueStore[K, V]) extends KeyValueSt
     store.delete(key)
   }
 
-  def deleteAll(keys: java.util.List[K]) = {
+  override def deleteAll(keys: java.util.List[K]) = {
     notNull(keys, NullKeysErrorMessage)
     keys.asScala.foreach(key => notNull(key, NullKeyErrorMessage))
     store.deleteAll(keys)
diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/SerializedKeyValueStore.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/SerializedKeyValueStore.scala
index c8939b7451..16dd980678 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/SerializedKeyValueStore.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/SerializedKeyValueStore.scala
@@ -38,7 +38,7 @@ class SerializedKeyValueStore[K, V](
     fromBytesOrNull(found, msgSerde)
   }
 
-  def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
+  override def getAll(keys: java.util.List[K]): java.util.Map[K, V] = {
     metrics.gets.inc(keys.size)
     val mapBytes = store.getAll(serializeKeys(keys))
     if (mapBytes != null) {
@@ -80,7 +80,7 @@ class SerializedKeyValueStore[K, V](
     store.delete(keyBytes)
   }
 
-  def deleteAll(keys: java.util.List[K]) = {
+  override def deleteAll(keys: java.util.List[K]) = {
     metrics.deletes.inc(keys.size)
     store.deleteAll(serializeKeys(keys))
   }

From e74998c5e58c891c0744ce1be2b13f7c5861656a Mon Sep 17 00:00:00 2001
From: Wei Song <wsong@linkedin.com>
Date: Tue, 5 Dec 2017 13:23:44 -0800
Subject: [PATCH 07/36] Initial version of Table API

Initial version of table API, it includes
 - Core table API (Table, TableDescriptor, TableSpec)
 - Local table implementation for in-memory and RocksDb
 - The writeTo() and stream-table join operators

nickpan47 xinyuiscool prateekm could you help review?

Author: Wei Song <wsong@linkedin.com>

Reviewers: Yi Pan <nickpan47@gmail.com>, Christopher Pettitt <cpettitt@linkedin.com>

Closes #349 from weisong44/table-api-14
---
 build.gradle                                  |   9 +
 .../java/org/apache/samza/operators/KV.java   |   2 +-
 .../apache/samza/operators/MessageStream.java |  51 ++-
 .../apache/samza/operators/StreamGraph.java   |  15 +
 .../samza/operators/TableDescriptor.java      |  73 +++++
 .../functions/StreamTableJoinFunction.java    |  59 ++++
 .../table/LocalStoreBackedTableProvider.java  |  37 +++
 .../apache/samza/table/ReadWriteTable.java    |  75 +++++
 .../org/apache/samza/table/ReadableTable.java |  61 ++++
 .../java/org/apache/samza/table/Table.java    |  31 ++
 .../org/apache/samza/table/TableProvider.java |  57 ++++
 .../samza/table/TableProviderFactory.java     |  35 ++
 .../org/apache/samza/table/TableSpec.java     | 125 +++++++
 .../org/apache/samza/task/TaskContext.java    |   6 +-
 .../apache/samza/config/JavaTableConfig.java  |  87 +++++
 .../samza/container/TaskContextImpl.java      |  24 +-
 .../samza/execution/ExecutionPlanner.java     |   5 +
 .../org/apache/samza/execution/JobGraph.java  |  16 +
 .../execution/JobGraphJsonGenerator.java      |  50 +++
 .../org/apache/samza/execution/JobNode.java   |  54 +++-
 .../samza/operators/BaseTableDescriptor.java  |  94 ++++++
 .../samza/operators/MessageStreamImpl.java    |  31 +-
 .../samza/operators/StreamGraphImpl.java      |  40 ++-
 .../org/apache/samza/operators/TableImpl.java |  40 +++
 .../operators/impl/OperatorImplGraph.java     |  30 +-
 .../impl/SendToTableOperatorImpl.java         |  71 ++++
 .../impl/StreamTableJoinOperatorImpl.java     |  82 +++++
 .../samza/operators/spec/OperatorSpec.java    |  12 +-
 .../samza/operators/spec/OperatorSpecs.java   |  44 ++-
 .../spec/SendToTableOperatorSpec.java         |  65 ++++
 .../spec/StreamTableJoinOperatorSpec.java     |  67 ++++
 .../org/apache/samza/table/TableManager.java  | 153 +++++++++
 .../samza/container/SamzaContainer.scala      |  75 ++---
 .../apache/samza/container/TaskInstance.scala |  40 ++-
 .../samza/config/TestJavaTableConfig.java     |  58 ++++
 .../operators/TestMessageStreamImpl.java      |  68 +++-
 .../samza/operators/TestStreamGraphImpl.java  |  25 +-
 .../impl/TestStreamTableJoinOperatorImpl.java | 101 ++++++
 .../apache/samza/table/TestTableManager.java  | 176 ++++++++++
 .../apache/samza/task/TestAsyncRunLoop.java   |  14 +-
 .../kv/inmemory/InMemoryTableDescriptor.java  |  59 ++++
 .../kv/inmemory/InMemoryTableProvider.java    |  65 ++++
 .../InMemoryTableProviderFactory.java         |  33 ++
 .../inmemory/TestInMemoryTableDescriptor.java |  48 +++
 .../inmemory/TestInMemoryTableProvider.java   |  65 ++++
 .../storage/kv/RocksDbTableDescriptor.java    | 232 +++++++++++++
 .../storage/kv/RocksDbTableProvider.java      |  64 ++++
 .../kv/RocksDbTableProviderFactory.java       |  31 ++
 .../kv/TestRocksDbTableDescriptor.java        |  87 +++++
 .../storage/kv/TestRocksDbTableProvider.java  |  66 ++++
 .../BaseLocalStoreBackedTableDescriptor.java  |  56 ++++
 .../kv/BaseLocalStoreBackedTableProvider.java |  92 ++++++
 .../kv/LocalStoreBackedReadWriteTable.java    |  68 ++++
 .../kv/LocalStoreBackedReadableTable.java     |  61 ++++
 ...TestLocalBaseStoreBackedTableProvider.java |  77 +++++
 .../samza/test/table/TestLocalTable.java      | 304 ++++++++++++++++++
 .../samza/test/table/TestTableData.java       | 200 ++++++++++++
 .../samza/test/util/ArraySystemConsumer.java  |   4 +-
 .../samza/test/util/SimpleSystemAdmin.java    |  26 +-
 59 files changed, 3668 insertions(+), 128 deletions(-)
 create mode 100644 samza-api/src/main/java/org/apache/samza/operators/TableDescriptor.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/operators/functions/StreamTableJoinFunction.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/LocalStoreBackedTableProvider.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/ReadWriteTable.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/ReadableTable.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/Table.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/TableProvider.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/TableProviderFactory.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/table/TableSpec.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/config/JavaTableConfig.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/operators/BaseTableDescriptor.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/operators/TableImpl.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/operators/impl/SendToTableOperatorImpl.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/operators/impl/StreamTableJoinOperatorImpl.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/operators/spec/SendToTableOperatorSpec.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/operators/spec/StreamTableJoinOperatorSpec.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/table/TableManager.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/config/TestJavaTableConfig.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/operators/impl/TestStreamTableJoinOperatorImpl.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/table/TestTableManager.java
 create mode 100644 samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableDescriptor.java
 create mode 100644 samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProvider.java
 create mode 100644 samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProviderFactory.java
 create mode 100644 samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableDescriptor.java
 create mode 100644 samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableProvider.java
 create mode 100644 samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableDescriptor.java
 create mode 100644 samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProvider.java
 create mode 100644 samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProviderFactory.java
 create mode 100644 samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableDescriptor.java
 create mode 100644 samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableProvider.java
 create mode 100644 samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableDescriptor.java
 create mode 100644 samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableProvider.java
 create mode 100644 samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadWriteTable.java
 create mode 100644 samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadableTable.java
 create mode 100644 samza-kv/src/test/java/org/apache/samza/storage/kv/TestLocalBaseStoreBackedTableProvider.java
 create mode 100644 samza-test/src/test/java/org/apache/samza/test/table/TestLocalTable.java
 create mode 100644 samza-test/src/test/java/org/apache/samza/test/table/TestTableData.java

diff --git a/build.gradle b/build.gradle
index 50cc5e0bdc..be1baf7b8d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -565,6 +565,15 @@ project(":samza-kv_$scalaVersion") {
 project(":samza-kv-inmemory_$scalaVersion") {
   apply plugin: 'scala'
 
+  // Force scala joint compilation
+  sourceSets.main.scala.srcDir "src/main/java"
+  sourceSets.test.scala.srcDir "src/test/java"
+
+  // Disable the Javac compiler by forcing joint compilation by scalac. This is equivalent to setting
+  // tasks.compileTestJava.enabled = false
+  sourceSets.main.java.srcDirs = []
+  sourceSets.test.java.srcDirs = []
+
   dependencies {
     compile project(':samza-api')
     compile project(":samza-core_$scalaVersion")
diff --git a/samza-api/src/main/java/org/apache/samza/operators/KV.java b/samza-api/src/main/java/org/apache/samza/operators/KV.java
index 0bed3b9cd5..824bcb49f1 100644
--- a/samza-api/src/main/java/org/apache/samza/operators/KV.java
+++ b/samza-api/src/main/java/org/apache/samza/operators/KV.java
@@ -25,7 +25,7 @@
  * @param <K> type of the key
  * @param <V> type of the value
  */
-public class KV<K, V> {
+public final class KV<K, V> {
   public final K key;
   public final V value;
 
diff --git a/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java b/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java
index acb2f33fc6..f0a5526a2b 100644
--- a/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java
+++ b/samza-api/src/main/java/org/apache/samza/operators/MessageStream.java
@@ -18,21 +18,23 @@
  */
 package org.apache.samza.operators;
 
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.function.Function;
+
 import org.apache.samza.annotation.InterfaceStability;
 import org.apache.samza.operators.functions.FilterFunction;
 import org.apache.samza.operators.functions.FlatMapFunction;
 import org.apache.samza.operators.functions.JoinFunction;
 import org.apache.samza.operators.functions.MapFunction;
 import org.apache.samza.operators.functions.SinkFunction;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
 import org.apache.samza.operators.windows.Window;
 import org.apache.samza.operators.windows.WindowPane;
 import org.apache.samza.serializers.KVSerde;
 import org.apache.samza.serializers.Serde;
-
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.function.Function;
+import org.apache.samza.table.Table;
 
 
 /**
@@ -153,6 +155,34 @@ <K, OM, JM> MessageStream<JM> join(MessageStream<OM> otherStream,
       Serde<K> keySerde, Serde<M> messageSerde, Serde<OM> otherMessageSerde,
       Duration ttl, String id);
 
+  /**
+   * Joins this {@link MessageStream} with another {@link Table} using the provided
+   * pairwise {@link StreamTableJoinFunction}.
+   * <p>
+   * The type of input message is expected to be {@link KV}.
+   * <p>
+   * Records are looked up from the joined table using the join key, join function
+   * is applied and join results are emitted as matches are found.
+   * <p>
+   * The join function allows implementation of both inner and left outer join. A null will be
+   * passed to the join function, if no record matching the join key is found in the table.
+   * The join function can choose to return an instance of JM (outer left join) or null
+   * (inner join); if null is returned, it won't be processed further.
+   * <p>
+   * Both the input stream and table being joined must have the same number of partitions,
+   * and should be partitioned by the same join key.
+   * <p>
+   *
+   * @param table the table being joined
+   * @param joinFn the join function
+   * @param <K> the type of join key
+   * @param <R> the type of table record
+   * @param <JM> the type of messages resulting from the {@code joinFn}
+   * @return the joined {@link MessageStream}
+   */
+  <K, R extends KV, JM> MessageStream<JM> join(Table<R> table,
+      StreamTableJoinFunction<? extends K, ? super M, ? super R, ? extends JM> joinFn);
+
   /**
    * Merges all {@code otherStreams} with this {@link MessageStream}.
    * <p>
@@ -235,4 +265,15 @@ <K, V> MessageStream<KV<K, V>> partitionBy(Function<? super M, ? extends K> keyE
    */
   <K, V> MessageStream<KV<K, V>> partitionBy(Function<? super M, ? extends K> keyExtractor,
       Function<? super M, ? extends V> valueExtractor, String id);
+
+  /**
+   * Sends messages in this {@link MessageStream} to a {@link Table}. The type of input message is expected
+   * to be {@link KV}, otherwise a {@link ClassCastException} will be thrown.
+   *
+   * @param table the table to write messages to
+   * @param <K> the type of key in the table
+   * @param <V> the type of record value in the table
+   */
+  <K, V> void sendTo(Table<KV<K, V>> table);
+
 }
diff --git a/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java b/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java
index 49306316db..6871bc76d4 100644
--- a/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java
+++ b/samza-api/src/main/java/org/apache/samza/operators/StreamGraph.java
@@ -20,6 +20,7 @@
 
 import org.apache.samza.annotation.InterfaceStability;
 import org.apache.samza.serializers.Serde;
+import org.apache.samza.table.Table;
 
 
 /**
@@ -120,6 +121,20 @@ public interface StreamGraph {
    */
   <M> OutputStream<M> getOutputStream(String streamId);
 
+  /**
+   * Gets the {@link Table} corresponding to the {@link TableDescriptor}.
+   * <p>
+   * Multiple invocations of this method with the same {@link TableDescriptor} will throw an
+   * {@link IllegalStateException}.
+   *
+   * @param tableDesc the {@link TableDescriptor}
+   * @param <K> the type of the key
+   * @param <V> the type of the value
+   * @return the {@link Table} corresponding to the {@code tableDesc}
+   * @throws IllegalStateException when invoked multiple times with the same {@link TableDescriptor}
+   */
+  <K, V> Table<KV<K, V>> getTable(TableDescriptor<K, V, ?> tableDesc);
+
   /**
    * Sets the {@link ContextManager} for this {@link StreamGraph}.
    * <p>
diff --git a/samza-api/src/main/java/org/apache/samza/operators/TableDescriptor.java b/samza-api/src/main/java/org/apache/samza/operators/TableDescriptor.java
new file mode 100644
index 0000000000..a60b6a9ecb
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/operators/TableDescriptor.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators;
+
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.serializers.KVSerde;
+
+/**
+ * User facing class to collect metadata that fully describes a
+ * Samza table. This interface should be implemented by concrete table implementations.
+ * <p>
+ * Typical user code should look like the following, notice <code>withConfig()</code>
+ * is defined in this class and the rest in subclasses.
+ *
+ * <pre>
+ * {@code
+ * TableDescriptor<Integer, String, ?> tableDesc = new RocksDbTableDescriptor("tbl")
+ *     .withSerde(KVSerde.of(new IntegerSerde(), new StringSerde("UTF-8")))
+ *     .withBlockSize(1024)
+ *     .withConfig("some-key", "some-value");
+ * }
+ * </pre>
+
+ * Once constructed, a table descriptor can be registered with the system. Internally,
+ * the table descriptor is then converted to a {@link org.apache.samza.table.TableSpec},
+ * which is used to track tables internally.
+ *
+ * @param <K> the type of the key in this table
+ * @param <V> the type of the value in this table
+ * @param <D> the type of the concrete table descriptor
+ */
+@InterfaceStability.Unstable
+public interface TableDescriptor<K, V, D extends TableDescriptor<K, V, D>> {
+
+  /**
+   * Get the Id of the table
+   * @return Id of the table
+   */
+  String getTableId();
+
+  /**
+   * Set the Serde for this table
+   * @param serde the serde
+   * @return this table descriptor instance
+   * @throws IllegalArgumentException if null is provided
+   */
+  D withSerde(KVSerde<K, V> serde);
+
+  /**
+   * Add a configuration entry for the table
+   * @param key the key
+   * @param value the value
+   * @return this table descriptor instance
+   */
+  D withConfig(String key, String value);
+
+}
\ No newline at end of file
diff --git a/samza-api/src/main/java/org/apache/samza/operators/functions/StreamTableJoinFunction.java b/samza-api/src/main/java/org/apache/samza/operators/functions/StreamTableJoinFunction.java
new file mode 100644
index 0000000000..6afcf67ef1
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/operators/functions/StreamTableJoinFunction.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators.functions;
+
+import org.apache.samza.annotation.InterfaceStability;
+
+
+/**
+ * Joins incoming messages with records from a table by the join key.
+ *
+ * @param <K> the type of join key
+ * @param <M>  type of input message
+ * @param <R>  type of the table record
+ * @param <JM> type of join results
+ */
+@InterfaceStability.Unstable
+public interface StreamTableJoinFunction<K, M, R, JM> extends InitableFunction, ClosableFunction {
+
+  /**
+   * Joins the provided messages and table record, returns the joined message.
+   *
+   * @param message  the input message
+   * @param record  the table record value
+   * @return  the join result
+   */
+  JM apply(M message, R record);
+
+  /**
+   * Retrieve the join key from incoming messages
+   *
+   * @param message incoming message
+   * @return the join key
+   */
+  K getMessageKey(M message);
+
+  /**
+   * Retrieve the join key from table record
+   *
+   * @param record table record
+   * @return the join key
+   */
+  K getRecordKey(R record);
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/LocalStoreBackedTableProvider.java b/samza-api/src/main/java/org/apache/samza/table/LocalStoreBackedTableProvider.java
new file mode 100644
index 0000000000..21630ab10f
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/LocalStoreBackedTableProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import org.apache.samza.storage.StorageEngine;
+
+
+/**
+ * Interface for tables backed by Samza local stores. The backing stores are
+ * injected during initialization of the table. Since the lifecycle
+ * of the underlying stores are already managed by Samza container,
+ * the table provider will not manage the lifecycle of the backing
+ * stores.
+ */
+public interface LocalStoreBackedTableProvider extends TableProvider {
+  /**
+   * Initializes the table provider with the backing store
+   * @param store the backing store
+   */
+  void init(StorageEngine store);
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/ReadWriteTable.java b/samza-api/src/main/java/org/apache/samza/table/ReadWriteTable.java
new file mode 100644
index 0000000000..d617153cbf
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/ReadWriteTable.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import java.util.List;
+
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.storage.kv.Entry;
+
+/**
+ *
+ * A table that supports get, put and delete by one or more keys
+ *
+ * @param <K> the type of the key in this table
+ * @param <V> the type of the value in this table
+ */
+@InterfaceStability.Unstable
+public interface ReadWriteTable<K, V> extends ReadableTable<K, V> {
+
+  /**
+   * Updates the mapping of the specified key-value pair; Associates the specified {@code key} with the specified {@code value}.
+   *
+   * @param key the key with which the specified {@code value} is to be associated.
+   * @param value the value with which the specified {@code key} is to be associated.
+   * @throws NullPointerException if the specified {@code key} or {@code value} is {@code null}.
+   */
+  void put(K key, V value);
+
+  /**
+   * Updates the mappings of the specified key-value {@code entries}.
+   *
+   * @param entries the updated mappings to put into this table.
+   * @throws NullPointerException if any of the specified {@code entries} has {@code null} as key or value.
+   */
+  void putAll(List<Entry<K, V>> entries);
+
+  /**
+   * Deletes the mapping for the specified {@code key} from this table (if such mapping exists).
+   *
+   * @param key the key for which the mapping is to be deleted.
+   * @throws NullPointerException if the specified {@code key} is {@code null}.
+   */
+  void delete(K key);
+
+  /**
+   * Deletes the mappings for the specified {@code keys} from this table.
+   *
+   * @param keys the keys for which the mappings are to be deleted.
+   * @throws NullPointerException if the specified {@code keys} list, or any of the keys, is {@code null}.
+   */
+  void deleteAll(List<K> keys);
+
+
+  /**
+   * Flushes the underlying store of this table, if applicable.
+   */
+  void flush();
+
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/ReadableTable.java b/samza-api/src/main/java/org/apache/samza/table/ReadableTable.java
new file mode 100644
index 0000000000..5ad6e0f627
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/ReadableTable.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.operators.KV;
+
+
+/**
+ *
+ * A table that supports get by one or more keys
+ *
+ * @param <K> the type of the record key in this table
+ * @param <V> the type of the record value in this table
+ */
+@InterfaceStability.Unstable
+public interface ReadableTable<K, V> extends Table<KV<K, V>> {
+
+  /**
+   * Gets the value associated with the specified {@code key}.
+   *
+   * @param key the key with which the associated value is to be fetched.
+   * @return if found, the value associated with the specified {@code key}; otherwise, {@code null}.
+   * @throws NullPointerException if the specified {@code key} is {@code null}.
+   */
+  V get(K key);
+
+  /**
+   * Gets the values with which the specified {@code keys} are associated.
+   *
+   * @param keys the keys with which the associated values are to be fetched.
+   * @return a map of the keys that were found and their respective values.
+   * @throws NullPointerException if the specified {@code keys} list, or any of the keys, is {@code null}.
+   */
+  Map<K, V> getAll(List<K> keys);
+
+  /**
+   * Close the table and release any resources acquired
+   */
+  void close();
+
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/Table.java b/samza-api/src/main/java/org/apache/samza/table/Table.java
new file mode 100644
index 0000000000..767e17697f
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/Table.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import org.apache.samza.annotation.InterfaceStability;
+
+/**
+ *
+ * Marker interface for a table.
+ *
+ * @param <R> the type of records in the table
+ */
+@InterfaceStability.Unstable
+public interface Table<R> {
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/TableProvider.java b/samza-api/src/main/java/org/apache/samza/table/TableProvider.java
new file mode 100644
index 0000000000..54c6f5d088
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/TableProvider.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import java.util.Map;
+
+import org.apache.samza.annotation.InterfaceStability;
+
+
+/**
+ * A table provider provides the implementation for a table. It ensures a table is
+ * properly constructed and also manages its lifecycle.
+ */
+@InterfaceStability.Unstable
+public interface TableProvider {
+  /**
+   * Get an instance of the table for read/write operations
+   * @return the underlying table
+   */
+  Table getTable();
+
+  /**
+   * Generate any configuration for this table, the generated configuration
+   * is used by Samza container to construct this table and any components
+   * necessary
+   * .
+   * @param config the current configuration
+   * @return configuration for this table
+   */
+  Map<String, String> generateConfig(Map<String, String> config);
+
+  /**
+   * Start the underlying table
+   */
+  void start();
+
+  /**
+   * Stop the underlying table
+   */
+  void stop();
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/TableProviderFactory.java b/samza-api/src/main/java/org/apache/samza/table/TableProviderFactory.java
new file mode 100644
index 0000000000..1bb0196717
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/TableProviderFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import org.apache.samza.annotation.InterfaceStability;
+
+
+/**
+ * Factory of a table provider object
+ */
+@InterfaceStability.Unstable
+public interface TableProviderFactory {
+  /**
+   * Constructs an instances of the table provider based on a given table spec
+   * @param tableSpec the table spec
+   * @return the table provider
+   */
+  TableProvider getTableProvider(TableSpec tableSpec);
+}
diff --git a/samza-api/src/main/java/org/apache/samza/table/TableSpec.java b/samza-api/src/main/java/org/apache/samza/table/TableSpec.java
new file mode 100644
index 0000000000..68043f98d8
--- /dev/null
+++ b/samza-api/src/main/java/org/apache/samza/table/TableSpec.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.serializers.KVSerde;
+
+
+/**
+ * TableSpec is a blueprint for creating, validating, or simply describing a table in the runtime environment.
+ *
+ * It is typically created indirectly by constructing an instance of {@link org.apache.samza.operators.TableDescriptor},
+ * and then invoke <code>BaseTableDescriptor.getTableSpec()</code>.
+ *
+ * It has specific attributes for common behaviors that Samza uses.
+ *
+ * It has the table provider factory, which provides the actual table implementation.
+ *
+ * It also includes a map of configurations which may be implementation-specific.
+ *
+ * It is immutable by design.
+ */
+@InterfaceStability.Unstable
+public class TableSpec {
+
+  private final String id;
+  private final KVSerde serde;
+  private final String tableProviderFactoryClassName;
+  private final Map<String, String> config = new HashMap<>();
+
+  /**
+   * Default constructor
+   */
+  public TableSpec() {
+    this.id = null;
+    this.serde = null;
+    this.tableProviderFactoryClassName = null;
+  }
+
+  /**
+   * Constructs a {@link TableSpec}
+   *
+   * @param tableId Id of the table
+   * @param tableProviderFactoryClassName table provider factory
+   * @param serde the serde
+   * @param config implementation specific configuration
+   */
+  public TableSpec(String tableId, KVSerde serde, String tableProviderFactoryClassName,
+      Map<String, String> config) {
+    this.id = tableId;
+    this.serde = serde;
+    this.tableProviderFactoryClassName = tableProviderFactoryClassName;
+    this.config.putAll(config);
+  }
+
+  /**
+   * Get the Id of the table
+   * @return Id of the table
+   */
+  public String getId() {
+    return id;
+  }
+
+  /**
+   * Get the serde
+   * @param <K> the type of the key
+   * @param <V> the type of the value
+   * @return the key serde
+   */
+  public <K, V> KVSerde<K, V> getSerde() {
+    return serde;
+  }
+
+  /**
+   * Get the class name of the table provider factory
+   * @return class name of the table provider factory
+   */
+  public String getTableProviderFactoryClassName() {
+    return tableProviderFactoryClassName;
+  }
+
+  /**
+   * Get implementation configuration for the table
+   * @return configuration for the table
+   */
+  public Map<String, String> getConfig() {
+    return Collections.unmodifiableMap(config);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || !getClass().equals(o.getClass())) {
+      return false;
+    }
+    return id.equals(((TableSpec) o).id);
+  }
+
+  @Override
+  public int hashCode() {
+    return id.hashCode();
+  }
+}
diff --git a/samza-api/src/main/java/org/apache/samza/task/TaskContext.java b/samza-api/src/main/java/org/apache/samza/task/TaskContext.java
index 4ef3d301b3..11ffacc35f 100644
--- a/samza-api/src/main/java/org/apache/samza/task/TaskContext.java
+++ b/samza-api/src/main/java/org/apache/samza/task/TaskContext.java
@@ -19,12 +19,14 @@
 
 package org.apache.samza.task;
 
+import java.util.Set;
+
 import org.apache.samza.container.SamzaContainerContext;
 import org.apache.samza.container.TaskName;
 import org.apache.samza.metrics.MetricsRegistry;
 import org.apache.samza.system.SystemStreamPartition;
+import org.apache.samza.table.Table;
 
-import java.util.Set;
 
 /**
  * A TaskContext provides resources about the {@link org.apache.samza.task.StreamTask}, particularly during
@@ -37,6 +39,8 @@ public interface TaskContext {
 
   Object getStore(String name);
 
+  Table getTable(String tableId);
+
   TaskName getTaskName();
 
   SamzaContainerContext getSamzaContainerContext();
diff --git a/samza-core/src/main/java/org/apache/samza/config/JavaTableConfig.java b/samza-core/src/main/java/org/apache/samza/config/JavaTableConfig.java
new file mode 100644
index 0000000000..6cc39869e9
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/config/JavaTableConfig.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.config;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A helper class for handling table configuration
+ */
+public class JavaTableConfig extends MapConfig {
+
+  // Prefix
+  public static final String TABLES_PREFIX = "tables.";
+  public static final String TABLE_ID_PREFIX = TABLES_PREFIX + "%s";
+
+  // Suffix
+  public static final String TABLE_PROVIDER_FACTORY_SUFFIX = ".provider.factory";
+
+  // Config keys
+  public static final String TABLE_PROVIDER_FACTORY = String.format("%s.provider.factory", TABLE_ID_PREFIX);
+  public static final String TABLE_KEY_SERDE = String.format("%s.key.serde", TABLE_ID_PREFIX);
+  public static final String TABLE_VALUE_SERDE = String.format("%s.value.serde", TABLE_ID_PREFIX);
+
+
+  public JavaTableConfig(Config config) {
+    super(config);
+  }
+
+  /**
+   * Get Id's of all tables
+   * @return list of table Id's
+   */
+  public List<String> getTableIds() {
+    Config subConfig = subset(TABLES_PREFIX, true);
+    Set<String> tableNames = subConfig.keySet().stream()
+        .filter(k -> k.endsWith(TABLE_PROVIDER_FACTORY_SUFFIX))
+        .map(k -> k.substring(0, k.indexOf(".")))
+        .collect(Collectors.toSet());
+    return new LinkedList<>(tableNames);
+  }
+
+  /**
+   * Get the {@link org.apache.samza.table.TableProviderFactory} class for a table
+   * @param tableId Id of the table
+   * @return the {@link org.apache.samza.table.TableProviderFactory} class name
+   */
+  public String getTableProviderFactory(String tableId) {
+    return get(String.format(TABLE_PROVIDER_FACTORY, tableId), null);
+  }
+
+  /**
+   * Get registry keys of key serde for this table
+   * @param tableId Id of the table
+   * @return serde retistry key
+   */
+  public String getKeySerde(String tableId) {
+    return get(String.format(TABLE_KEY_SERDE, tableId), null);
+  }
+
+  /**
+   * Get registry keys of value serde for this table
+   * @param tableId Id of the table
+   * @return serde retistry key
+   */
+  public String getValueSerde(String tableId) {
+    return get(String.format(TABLE_VALUE_SERDE, tableId), null);
+  }
+}
diff --git a/samza-core/src/main/java/org/apache/samza/container/TaskContextImpl.java b/samza-core/src/main/java/org/apache/samza/container/TaskContextImpl.java
index aa622a362f..0248486431 100644
--- a/samza-core/src/main/java/org/apache/samza/container/TaskContextImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/container/TaskContextImpl.java
@@ -19,7 +19,10 @@
 
 package org.apache.samza.container;
 
-import com.google.common.collect.ImmutableSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.samza.checkpoint.OffsetManager;
 import org.apache.samza.job.model.JobModel;
 import org.apache.samza.metrics.ReadableMetricsRegistry;
@@ -27,13 +30,13 @@
 import org.apache.samza.storage.kv.KeyValueStore;
 import org.apache.samza.system.StreamMetadataCache;
 import org.apache.samza.system.SystemStreamPartition;
+import org.apache.samza.table.Table;
+import org.apache.samza.table.TableManager;
 import org.apache.samza.task.TaskContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.collect.ImmutableSet;
 
 public class TaskContextImpl implements TaskContext {
   private static final Logger LOG = LoggerFactory.getLogger(TaskContextImpl.class);
@@ -44,6 +47,7 @@ public class TaskContextImpl implements TaskContext {
   private final Set<SystemStreamPartition> systemStreamPartitions;
   private final OffsetManager offsetManager;
   private final TaskStorageManager storageManager;
+  private final TableManager tableManager;
   private final JobModel jobModel;
   private final StreamMetadataCache streamMetadataCache;
   private final Map<String, Object> objectRegistry = new HashMap<>();
@@ -56,6 +60,7 @@ public TaskContextImpl(TaskName taskName,
                          Set<SystemStreamPartition> systemStreamPartitions,
                          OffsetManager offsetManager,
                          TaskStorageManager storageManager,
+                         TableManager tableManager,
                          JobModel jobModel,
                          StreamMetadataCache streamMetadataCache) {
     this.taskName = taskName;
@@ -64,6 +69,7 @@ public TaskContextImpl(TaskName taskName,
     this.systemStreamPartitions = ImmutableSet.copyOf(systemStreamPartitions);
     this.offsetManager = offsetManager;
     this.storageManager = storageManager;
+    this.tableManager = tableManager;
     this.jobModel = jobModel;
     this.streamMetadataCache = streamMetadataCache;
   }
@@ -88,6 +94,16 @@ public KeyValueStore getStore(String storeName) {
     }
   }
 
+  @Override
+  public Table getTable(String tableId) {
+    if (tableManager != null) {
+      return tableManager.getTable(tableId);
+    } else {
+      LOG.warn("No table manager found");
+      return null;
+    }
+  }
+
   @Override
   public TaskName getTaskName() {
     return taskName;
diff --git a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java
index 468aab97f0..e2c122a426 100644
--- a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java
+++ b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java
@@ -39,6 +39,7 @@
 import org.apache.samza.operators.spec.OperatorSpec;
 import org.apache.samza.system.StreamSpec;
 import org.apache.samza.system.SystemStream;
+import org.apache.samza.table.TableSpec;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -95,6 +96,7 @@ private void validateConfig() {
     Set<StreamSpec> sourceStreams = new HashSet<>(streamGraph.getInputOperators().keySet());
     Set<StreamSpec> sinkStreams = new HashSet<>(streamGraph.getOutputStreams().keySet());
     Set<StreamSpec> intStreams = new HashSet<>(sourceStreams);
+    Set<TableSpec> tables = new HashSet<>(streamGraph.getTables().keySet());
     intStreams.retainAll(sinkStreams);
     sourceStreams.removeAll(intStreams);
     sinkStreams.removeAll(intStreams);
@@ -113,6 +115,9 @@ private void validateConfig() {
     // add intermediate streams
     intStreams.forEach(spec -> jobGraph.addIntermediateStream(spec, node, node));
 
+    // add tables
+    tables.forEach(spec -> jobGraph.addTable(spec, node));
+
     jobGraph.validate();
 
     return jobGraph;
diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobGraph.java b/samza-core/src/main/java/org/apache/samza/execution/JobGraph.java
index 2a09e90833..4a09260387 100644
--- a/samza-core/src/main/java/org/apache/samza/execution/JobGraph.java
+++ b/samza-core/src/main/java/org/apache/samza/execution/JobGraph.java
@@ -30,11 +30,13 @@
 import java.util.Queue;
 import java.util.Set;
 import java.util.stream.Collectors;
+
 import org.apache.samza.config.ApplicationConfig;
 import org.apache.samza.config.Config;
 import org.apache.samza.config.JobConfig;
 import org.apache.samza.operators.StreamGraphImpl;
 import org.apache.samza.system.StreamSpec;
+import org.apache.samza.table.TableSpec;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,6 +57,7 @@
   private final Set<StreamEdge> sources = new HashSet<>();
   private final Set<StreamEdge> sinks = new HashSet<>();
   private final Set<StreamEdge> intermediateStreams = new HashSet<>();
+  private final Set<TableSpec> tables = new HashSet<>();
   private final Config config;
   private final JobGraphJsonGenerator jsonGenerator = new JobGraphJsonGenerator();
 
@@ -86,6 +89,11 @@ public List<StreamSpec> getIntermediateStreams() {
         .collect(Collectors.toList());
   }
 
+  void addTable(TableSpec tableSpec, JobNode node) {
+    tables.add(tableSpec);
+    node.addTable(tableSpec);
+  }
+
   @Override
   public String getPlanAsJson() throws Exception {
     return jsonGenerator.toJson(this);
@@ -210,6 +218,14 @@ Set<StreamEdge> getSinks() {
     return Collections.unmodifiableSet(sinks);
   }
 
+  /**
+   * Return the tables in the graph
+   * @return unmodifiable set of {@link TableSpec}
+   */
+  Set<TableSpec> getTables() {
+    return Collections.unmodifiableSet(tables);
+  }
+
   /**
    * Return the intermediate streams in the graph
    * @return unmodifiable set of {@link StreamEdge}
diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
index 03845e3854..2729fa3598 100644
--- a/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
+++ b/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
@@ -28,12 +28,15 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+
 import org.apache.samza.config.ApplicationConfig;
 import org.apache.samza.operators.spec.JoinOperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec.OpCode;
 import org.apache.samza.operators.spec.OutputOperatorSpec;
 import org.apache.samza.operators.spec.OutputStreamImpl;
+import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
+import org.apache.samza.table.TableSpec;
 import org.codehaus.jackson.annotate.JsonProperty;
 import org.codehaus.jackson.map.ObjectMapper;
 
@@ -54,6 +57,15 @@ static final class StreamSpecJson {
     int partitionCount;
   }
 
+  static final class TableSpecJson {
+    @JsonProperty("id")
+    String id;
+    @JsonProperty("tableProviderFactory")
+    String tableProviderFactory;
+    @JsonProperty("config")
+    Map<String, String> config;
+  }
+
   static final class StreamEdgeJson {
     @JsonProperty("streamSpec")
     StreamSpecJson streamSpec;
@@ -97,6 +109,8 @@ static final class JobGraphJson {
     Map<String, StreamEdgeJson> sinkStreams;
     @JsonProperty("intermediateStreams")
     Map<String, StreamEdgeJson> intermediateStreams;
+    @JsonProperty("tables")
+    Map<String, TableSpecJson> tables;
     @JsonProperty("applicationName")
     String applicationName;
     @JsonProperty("applicationId")
@@ -119,9 +133,11 @@ static final class JobGraphJson {
     jobGraphJson.sourceStreams = new HashMap<>();
     jobGraphJson.sinkStreams = new HashMap<>();
     jobGraphJson.intermediateStreams = new HashMap<>();
+    jobGraphJson.tables = new HashMap<>();
     jobGraph.getSources().forEach(e -> buildStreamEdgeJson(e, jobGraphJson.sourceStreams));
     jobGraph.getSinks().forEach(e -> buildStreamEdgeJson(e, jobGraphJson.sinkStreams));
     jobGraph.getIntermediateStreamEdges().forEach(e -> buildStreamEdgeJson(e, jobGraphJson.intermediateStreams));
+    jobGraph.getTables().forEach(t -> buildTableJson(t, jobGraphJson.tables));
 
     jobGraphJson.jobs = jobGraph.getJobNodes().stream()
         .map(jobNode -> buildJobNodeJson(jobNode))
@@ -206,6 +222,11 @@ private Map<String, Object> operatorToMap(OperatorSpec spec) {
       map.put("outputStreamId", outputStream.getStreamSpec().getId());
     }
 
+    if (spec instanceof StreamTableJoinOperatorSpec) {
+      TableSpec tableSpec = ((StreamTableJoinOperatorSpec) spec).getTableSpec();
+      map.put("tableId", tableSpec.getId());
+    }
+
     if (spec instanceof JoinOperatorSpec) {
       map.put("ttlMs", ((JoinOperatorSpec) spec).getTtlMs());
     }
@@ -247,4 +268,33 @@ private StreamEdgeJson buildStreamEdgeJson(StreamEdge edge, Map<String, StreamEd
     }
     return edgeJson;
   }
+
+  /**
+   * Get or create the JSON POJO for a {@link TableSpec}
+   * @param tableSpec the {@link TableSpec}
+   * @param tableSpecs a map of tableId to {@link TableSpecJson}
+   * @return JSON representation of the {@link TableSpec}
+   */
+  private TableSpecJson buildTableJson(TableSpec tableSpec, Map<String, TableSpecJson> tableSpecs) {
+    String tableId = tableSpec.getId();
+    TableSpecJson tableSpecJson = tableSpecs.get(tableId);
+    if (tableSpecJson == null) {
+      tableSpecJson = buildTableJson(tableSpec);
+      tableSpecs.put(tableId, tableSpecJson);
+    }
+    return tableSpecJson;
+  }
+
+  /**
+   * Create the JSON POJO for a {@link TableSpec}
+   * @param tableSpec the {@link TableSpec}
+   * @return JSON representation of the {@link TableSpec}
+   */
+  private TableSpecJson buildTableJson(TableSpec tableSpec) {
+    TableSpecJson tableSpecJson = new TableSpecJson();
+    tableSpecJson.id = tableSpec.getId();
+    tableSpecJson.tableProviderFactory = tableSpec.getTableProviderFactoryClassName();
+    tableSpecJson.config = tableSpec.getConfig();
+    return tableSpecJson;
+  }
 }
diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobNode.java b/samza-core/src/main/java/org/apache/samza/execution/JobNode.java
index 2e89292829..4e337d9637 100644
--- a/samza-core/src/main/java/org/apache/samza/execution/JobNode.java
+++ b/samza-core/src/main/java/org/apache/samza/execution/JobNode.java
@@ -19,7 +19,6 @@
 
 package org.apache.samza.execution;
 
-import com.google.common.base.Joiner;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Collection;
@@ -29,7 +28,9 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.stream.Collectors;
+
 import org.apache.samza.config.Config;
+import org.apache.samza.config.JavaTableConfig;
 import org.apache.samza.config.JobConfig;
 import org.apache.samza.config.MapConfig;
 import org.apache.samza.config.SerializerConfig;
@@ -47,10 +48,16 @@
 import org.apache.samza.serializers.Serde;
 import org.apache.samza.serializers.SerializableSerde;
 import org.apache.samza.system.StreamSpec;
+import org.apache.samza.table.TableProvider;
+import org.apache.samza.table.TableProviderFactory;
+import org.apache.samza.table.TableSpec;
 import org.apache.samza.util.Util;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Joiner;
+
+
 /**
  * A JobNode is a physical execution unit. In RemoteExecutionEnvironment, it's a job that will be submitted
  * to remote cluster. In LocalExecutionEnvironment, it's a set of StreamProcessors for local execution.
@@ -67,6 +74,7 @@ public class JobNode {
   private final StreamGraphImpl streamGraph;
   private final List<StreamEdge> inEdges = new ArrayList<>();
   private final List<StreamEdge> outEdges = new ArrayList<>();
+  private final List<TableSpec> tables = new ArrayList<>();
   private final Config config;
 
   JobNode(String jobName, String jobId, StreamGraphImpl streamGraph, Config config) {
@@ -109,6 +117,10 @@ List<StreamEdge> getOutEdges() {
     return outEdges;
   }
 
+  void addTable(TableSpec tableSpec) {
+    tables.add(tableSpec);
+  }
+
   /**
    * Generate the configs for a job
    * @param executionPlanJson JSON representation of the execution plan
@@ -147,6 +159,19 @@ public JobConfig generateConfig(String executionPlanJson) {
     // write serialized serde instances and stream serde configs to configs
     addSerdeConfigs(configs);
 
+    tables.forEach(tableSpec -> {
+        // Table provider factory
+        configs.put(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableSpec.getId()),
+            tableSpec.getTableProviderFactoryClassName());
+
+        // Note: no need to generate config for Serde's, as they are already produced by addSerdeConfigs()
+
+        // Generate additional configuration
+        TableProviderFactory tableProviderFactory = Util.getObj(tableSpec.getTableProviderFactoryClassName());
+        TableProvider tableProvider = tableProviderFactory.getTableProvider(tableSpec);
+        configs.putAll(tableProvider.generateConfig(configs));
+      });
+
     log.info("Job {} has generated configs {}", jobName, configs);
 
     String configPrefix = String.format(CONFIG_JOB_PREFIX, jobName);
@@ -209,11 +234,21 @@ void addSerdeConfigs(Map<String, String> configs) {
         }
       });
 
+    // collect all key and msg serde instances for tables
+    Map<String, Serde> tableKeySerdes = new HashMap<>();
+    Map<String, Serde> tableValueSerdes = new HashMap<>();
+    tables.forEach(tableSpec -> {
+        tableKeySerdes.put(tableSpec.getId(), tableSpec.getSerde().getKeySerde());
+        tableValueSerdes.put(tableSpec.getId(), tableSpec.getSerde().getValueSerde());
+      });
+
     // for each unique stream or store serde instance, generate a unique name and serialize to config
     HashSet<Serde> serdes = new HashSet<>(streamKeySerdes.values());
     serdes.addAll(streamMsgSerdes.values());
     serdes.addAll(storeKeySerdes.values());
     serdes.addAll(storeMsgSerdes.values());
+    serdes.addAll(tableKeySerdes.values());
+    serdes.addAll(tableValueSerdes.values());
     SerializableSerde<Serde> serializableSerde = new SerializableSerde<>();
     Base64.Encoder base64Encoder = Base64.getEncoder();
     Map<Serde, String> serdeUUIDs = new HashMap<>();
@@ -247,6 +282,17 @@ void addSerdeConfigs(Map<String, String> configs) {
         String msgSerdeConfigKey = String.format(StorageConfig.MSG_SERDE(), storeName);
         configs.put(msgSerdeConfigKey, serdeUUIDs.get(serde));
       });
+
+    // set key and msg serdes for tables to the serde names generated above
+    tableKeySerdes.forEach((tableId, serde) -> {
+        String keySerdeConfigKey = String.format(JavaTableConfig.TABLE_KEY_SERDE, tableId);
+        configs.put(keySerdeConfigKey, serdeUUIDs.get(serde));
+      });
+
+    tableValueSerdes.forEach((tableId, serde) -> {
+        String valueSerdeConfigKey = String.format(JavaTableConfig.TABLE_VALUE_SERDE, tableId);
+        configs.put(valueSerdeConfigKey, serdeUUIDs.get(serde));
+      });
   }
 
   /**
@@ -264,10 +310,14 @@ private long computeTriggerInterval() {
 
     // Filter out the join operators, and obtain a list of their ttl values
     List<Long> joinTtlIntervals = operatorSpecs.stream()
-        .filter(spec -> spec.getOpCode() == OperatorSpec.OpCode.JOIN)
+        .filter(spec -> spec instanceof JoinOperatorSpec)
         .map(spec -> ((JoinOperatorSpec) spec).getTtlMs())
         .collect(Collectors.toList());
 
+    if (joinTtlIntervals.isEmpty()) {
+      return -1;
+    }
+
     // Combine both the above lists
     List<Long> candidateTimerIntervals = new ArrayList<>(joinTtlIntervals);
     candidateTimerIntervals.addAll(windowTimerIntervals);
diff --git a/samza-core/src/main/java/org/apache/samza/operators/BaseTableDescriptor.java b/samza-core/src/main/java/org/apache/samza/operators/BaseTableDescriptor.java
new file mode 100644
index 0000000000..b875c2e52a
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/operators/BaseTableDescriptor.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.operators;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.serializers.NoOpSerde;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * Base class for all table descriptor implementations.
+ *
+ * @param <K> the type of the key in this table
+ * @param <V> the type of the value in this table
+ * @param <D> the type of the concrete table descriptor
+ */
+abstract public class BaseTableDescriptor<K, V, D extends BaseTableDescriptor<K, V, D>>
+    implements TableDescriptor<K, V, D> {
+
+  protected final String tableId;
+
+  protected KVSerde<K, V> serde = KVSerde.of(new NoOpSerde(), new NoOpSerde());
+
+  protected final Map<String, String> config = new HashMap<>();
+
+  /**
+   * Constructs a table descriptor instance
+   * @param tableId Id of the table
+   */
+  protected BaseTableDescriptor(String tableId) {
+    this.tableId = tableId;
+  }
+
+  @Override
+  public D withConfig(String key, String value) {
+    config.put(key, value);
+    return (D) this;
+  }
+
+  @Override
+  public D withSerde(KVSerde<K, V> serde) {
+    if (serde == null) {
+      throw new IllegalArgumentException("Serde cannot be null");
+    }
+    this.serde = serde;
+    return (D) this;
+  }
+
+  @Override
+  public String getTableId() {
+    return tableId;
+  }
+
+  /**
+   * Generate config for {@link TableSpec}; this method is used internally.
+   * @param tableSpecConfig configuration for the {@link TableSpec}
+   */
+  protected void generateTableSpecConfig(Map<String, String> tableSpecConfig) {
+    tableSpecConfig.putAll(config);
+  }
+
+  /**
+   * Validate that this table descriptor is constructed properly; this method is used internally.
+   */
+  protected void validate() {
+  }
+
+  /**
+   * Create a {@link TableSpec} from this table descriptor; this method is used internally.
+   *
+   * @return the {@link TableSpec}
+   */
+  abstract public TableSpec getTableSpec();
+}
diff --git a/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java b/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java
index 3f4e40d45e..07af54f6ac 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/MessageStreamImpl.java
@@ -19,12 +19,17 @@
 
 package org.apache.samza.operators;
 
+import java.time.Duration;
+import java.util.Collection;
+import java.util.function.Function;
+
 import org.apache.samza.SamzaException;
 import org.apache.samza.operators.functions.FilterFunction;
 import org.apache.samza.operators.functions.FlatMapFunction;
 import org.apache.samza.operators.functions.JoinFunction;
 import org.apache.samza.operators.functions.MapFunction;
 import org.apache.samza.operators.functions.SinkFunction;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
 import org.apache.samza.operators.spec.JoinOperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec.OpCode;
@@ -32,18 +37,18 @@
 import org.apache.samza.operators.spec.OutputOperatorSpec;
 import org.apache.samza.operators.spec.OutputStreamImpl;
 import org.apache.samza.operators.spec.PartitionByOperatorSpec;
+import org.apache.samza.operators.spec.SendToTableOperatorSpec;
 import org.apache.samza.operators.spec.SinkOperatorSpec;
 import org.apache.samza.operators.spec.StreamOperatorSpec;
+import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
 import org.apache.samza.operators.stream.IntermediateMessageStreamImpl;
 import org.apache.samza.operators.windows.Window;
 import org.apache.samza.operators.windows.WindowPane;
 import org.apache.samza.operators.windows.internal.WindowInternal;
 import org.apache.samza.serializers.KVSerde;
 import org.apache.samza.serializers.Serde;
-
-import java.time.Duration;
-import java.util.Collection;
-import java.util.function.Function;
+import org.apache.samza.table.Table;
+import org.apache.samza.table.TableSpec;
 
 
 /**
@@ -137,6 +142,16 @@ public <K, OM, JM> MessageStream<JM> join(MessageStream<OM> otherStream,
     return new MessageStreamImpl<>(this.graph, joinOpSpec);
   }
 
+  @Override
+  public <K, R extends KV, JM> MessageStream<JM> join(Table<R> table,
+      StreamTableJoinFunction<? extends K, ? super M, ? super R, ? extends JM> joinFn) {
+    TableSpec tableSpec = ((TableImpl) table).getTableSpec();
+    StreamTableJoinOperatorSpec<K, M, R, JM> joinOpSpec = OperatorSpecs.createStreamTableJoinOperatorSpec(
+        tableSpec, (StreamTableJoinFunction<K, M, R, JM>) joinFn, this.graph.getNextOpId(OpCode.JOIN));
+    this.operatorSpec.registerNextOperatorSpec(joinOpSpec);
+    return new MessageStreamImpl<>(this.graph, joinOpSpec);
+  }
+
   @Override
   public MessageStream<M> merge(Collection<? extends MessageStream<? extends M>> otherStreams) {
     if (otherStreams.isEmpty()) return this;
@@ -176,4 +191,12 @@ public <K, V> MessageStream<KV<K, V>> partitionBy(Function<? super M, ? extends
   protected OperatorSpec<?, M> getOperatorSpec() {
     return this.operatorSpec;
   }
+
+  @Override
+  public <K, V> void sendTo(Table<KV<K, V>> table) {
+    SendToTableOperatorSpec<K, V> op = OperatorSpecs.createSendToTableOperatorSpec(
+        this.operatorSpec, ((TableImpl) table).getTableSpec(), this.graph.getNextOpId(OpCode.SEND_TO));
+    this.operatorSpec.registerNextOperatorSpec(op);
+  }
+
 }
diff --git a/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java b/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java
index d014cb9419..b607c62f93 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/StreamGraphImpl.java
@@ -18,7 +18,15 @@
  */
 package org.apache.samza.operators;
 
-import com.google.common.base.Preconditions;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
@@ -34,17 +42,12 @@
 import org.apache.samza.serializers.NoOpSerde;
 import org.apache.samza.serializers.Serde;
 import org.apache.samza.system.StreamSpec;
+import org.apache.samza.table.Table;
+import org.apache.samza.table.TableSpec;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
+import com.google.common.base.Preconditions;
 
 /**
  * A {@link StreamGraph} that provides APIs for accessing {@link MessageStream}s to be used to
@@ -57,6 +60,7 @@ public class StreamGraphImpl implements StreamGraph {
   // We use a LHM for deterministic order in initializing and closing operators.
   private final Map<StreamSpec, InputOperatorSpec> inputOperators = new LinkedHashMap<>();
   private final Map<StreamSpec, OutputStreamImpl> outputStreams = new LinkedHashMap<>();
+  private final Map<TableSpec, TableImpl> tables = new LinkedHashMap<>();
   private final ApplicationRunner runner;
   private final Config config;
 
@@ -145,6 +149,18 @@ public <M> OutputStream<M> getOutputStream(String streamId) {
     return (OutputStream<M>) getOutputStream(streamId, defaultSerde);
   }
 
+  @Override
+  public <K, V> Table<KV<K, V>> getTable(TableDescriptor<K, V, ?> tableDesc) {
+    TableSpec tableSpec = ((BaseTableDescriptor) tableDesc).getTableSpec();
+    if (tables.containsKey(tableSpec)) {
+      throw new IllegalStateException(String.format(
+          "getTable() invoked multiple times with the same tableId: %s",
+          tableDesc.getTableId()));
+    }
+    tables.put(tableSpec, new TableImpl(tableSpec));
+    return tables.get(tableSpec);
+  }
+
   @Override
   public StreamGraph withContextManager(ContextManager contextManager) {
     this.contextManager = contextManager;
@@ -163,7 +179,7 @@ public StreamGraph withContextManager(ContextManager contextManager) {
    */
   <M> IntermediateMessageStreamImpl<M> getIntermediateStream(String streamId, Serde<M> serde) {
     StreamSpec streamSpec = runner.getStreamSpec(streamId);
-    
+
     Preconditions.checkState(!inputOperators.containsKey(streamSpec) && !outputStreams.containsKey(streamSpec),
         "getIntermediateStream must not be called multiple times with the same streamId: " + streamId);
 
@@ -190,6 +206,10 @@ public Map<StreamSpec, OutputStreamImpl> getOutputStreams() {
     return Collections.unmodifiableMap(outputStreams);
   }
 
+  public Map<TableSpec, TableImpl> getTables() {
+    return Collections.unmodifiableMap(tables);
+  }
+
   public ContextManager getContextManager() {
     return this.contextManager;
   }
diff --git a/samza-core/src/main/java/org/apache/samza/operators/TableImpl.java b/samza-core/src/main/java/org/apache/samza/operators/TableImpl.java
new file mode 100644
index 0000000000..e671534c4b
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/operators/TableImpl.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators;
+
+import org.apache.samza.table.Table;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * This class is the holder of a {@link TableSpec}
+ */
+public class TableImpl implements Table {
+
+  private final TableSpec tableSpec;
+
+  public TableImpl(TableSpec tableSpec) {
+    this.tableSpec = tableSpec;
+  }
+
+  public TableSpec getTableSpec() {
+    return tableSpec;
+  }
+
+}
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java
index 0bb12d23fd..ea278c16f4 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java
@@ -18,10 +18,15 @@
  */
 package org.apache.samza.operators.impl;
 
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
+
 import org.apache.samza.config.Config;
 import org.apache.samza.container.TaskContextImpl;
 import org.apache.samza.job.model.JobModel;
@@ -29,6 +34,7 @@
 import org.apache.samza.operators.StreamGraphImpl;
 import org.apache.samza.operators.functions.JoinFunction;
 import org.apache.samza.operators.functions.PartialJoinFunction;
+import org.apache.samza.operators.impl.store.TimestampedValue;
 import org.apache.samza.operators.spec.InputOperatorSpec;
 import org.apache.samza.operators.spec.JoinOperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec;
@@ -36,8 +42,9 @@
 import org.apache.samza.operators.spec.PartitionByOperatorSpec;
 import org.apache.samza.operators.spec.SinkOperatorSpec;
 import org.apache.samza.operators.spec.StreamOperatorSpec;
+import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
 import org.apache.samza.operators.spec.WindowOperatorSpec;
-import org.apache.samza.operators.impl.store.TimestampedValue;
+import org.apache.samza.operators.spec.SendToTableOperatorSpec;
 import org.apache.samza.storage.kv.KeyValueStore;
 import org.apache.samza.system.SystemStream;
 import org.apache.samza.task.TaskContext;
@@ -45,14 +52,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 
 /**
  * The DAG of {@link OperatorImpl}s corresponding to the DAG of {@link OperatorSpec}s.
@@ -212,6 +214,10 @@ OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec oper
       return new WindowOperatorImpl((WindowOperatorSpec) operatorSpec, clock);
     } else if (operatorSpec instanceof JoinOperatorSpec) {
       return createPartialJoinOperatorImpl(prevOperatorSpec, (JoinOperatorSpec) operatorSpec, config, context, clock);
+    } else if (operatorSpec instanceof StreamTableJoinOperatorSpec) {
+      return new StreamTableJoinOperatorImpl((StreamTableJoinOperatorSpec) operatorSpec, config, context);
+    } else if (operatorSpec instanceof SendToTableOperatorSpec) {
+      return new SendToTableOperatorImpl((SendToTableOperatorSpec) operatorSpec, config, context);
     }
     throw new IllegalArgumentException(
         String.format("Unsupported OperatorSpec: %s", operatorSpec.getClass().getName()));
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/SendToTableOperatorImpl.java b/samza-core/src/main/java/org/apache/samza/operators/impl/SendToTableOperatorImpl.java
new file mode 100644
index 0000000000..5ce13286c3
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/SendToTableOperatorImpl.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.samza.config.Config;
+import org.apache.samza.operators.KV;
+import org.apache.samza.operators.spec.OperatorSpec;
+import org.apache.samza.operators.spec.SendToTableOperatorSpec;
+import org.apache.samza.table.ReadWriteTable;
+import org.apache.samza.task.MessageCollector;
+import org.apache.samza.task.TaskContext;
+import org.apache.samza.task.TaskCoordinator;
+
+
+/**
+ * Implementation of a send-stream-to-table operator that stores the record
+ * in the table.
+ *
+ * @param <K> the type of the record key
+ * @param <V> the type of the record value
+ */
+public class SendToTableOperatorImpl<K, V> extends OperatorImpl<KV<K, V>, Void> {
+
+  private final SendToTableOperatorSpec<K, V> sendToTableOpSpec;
+  private final ReadWriteTable<K, V> table;
+
+  SendToTableOperatorImpl(SendToTableOperatorSpec<K, V> sendToTableOpSpec, Config config, TaskContext context) {
+    this.sendToTableOpSpec = sendToTableOpSpec;
+    this.table = (ReadWriteTable) context.getTable(sendToTableOpSpec.getTableSpec().getId());
+  }
+
+  @Override
+  protected void handleInit(Config config, TaskContext context) {
+  }
+
+  @Override
+  protected Collection<Void> handleMessage(KV<K, V> message, MessageCollector collector, TaskCoordinator coordinator) {
+    table.put(message.getKey(), message.getValue());
+    // there should be no further chained operators since this is a terminal operator.
+    return Collections.emptyList();
+  }
+
+  @Override
+  protected void handleClose() {
+    table.close();
+  }
+
+  @Override
+  protected OperatorSpec<KV<K, V>, Void> getOperatorSpec() {
+    return sendToTableOpSpec;
+  }
+}
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/StreamTableJoinOperatorImpl.java b/samza-core/src/main/java/org/apache/samza/operators/impl/StreamTableJoinOperatorImpl.java
new file mode 100644
index 0000000000..54a577088f
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/StreamTableJoinOperatorImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.samza.config.Config;
+import org.apache.samza.operators.KV;
+import org.apache.samza.operators.spec.OperatorSpec;
+import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
+import org.apache.samza.table.ReadableTable;
+import org.apache.samza.task.MessageCollector;
+import org.apache.samza.task.TaskContext;
+import org.apache.samza.task.TaskCoordinator;
+
+
+/**
+ * Implementation of a stream-table join operator that first retrieve the value of
+ * the message key from incoming message, and then apply the join function.
+ *
+ * @param <K> type of the join key
+ * @param <M> type of input messages
+ * @param <R> type of the table record
+ * @param <JM> type of the join result
+ */
+class StreamTableJoinOperatorImpl<K, M, R extends KV, JM> extends OperatorImpl<M, JM> {
+
+  private final StreamTableJoinOperatorSpec<K, M, R, JM> joinOpSpec;
+  private final ReadableTable<K, ?> table;
+
+  StreamTableJoinOperatorImpl(StreamTableJoinOperatorSpec<K, M, R, JM> joinOpSpec,
+      Config config, TaskContext context) {
+    this.joinOpSpec = joinOpSpec;
+    this.table = (ReadableTable) context.getTable(joinOpSpec.getTableSpec().getId());
+  }
+
+  @Override
+  protected void handleInit(Config config, TaskContext context) {
+    this.joinOpSpec.getJoinFn().init(config, context);
+  }
+
+  @Override
+  public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) {
+    K key = joinOpSpec.getJoinFn().getMessageKey(message);
+    Object recordValue = table.get(key);
+    R record = recordValue != null ? (R) KV.of(key, recordValue) : null;
+    JM output = joinOpSpec.getJoinFn().apply(message, record);
+
+    // The support for inner and outer join will be provided in the jonFn. For inner join, the joinFn might
+    // return null, when the corresponding record is absent in the table.
+    return output != null ?
+        Collections.singletonList(output)
+      : Collections.emptyList();
+  }
+
+  @Override
+  protected void handleClose() {
+    this.joinOpSpec.getJoinFn().close();
+  }
+
+  protected OperatorSpec<M, JM> getOperatorSpec() {
+    return joinOpSpec;
+  }
+
+}
diff --git a/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpec.java b/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpec.java
index 17f1b492cf..2a5991c6c7 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpec.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpec.java
@@ -18,17 +18,17 @@
  */
 package org.apache.samza.operators.spec;
 
-import org.apache.samza.annotation.InterfaceStability;
-import org.apache.samza.operators.functions.WatermarkFunction;
-import org.apache.samza.operators.MessageStream;
-import org.apache.samza.operators.MessageStreamImpl;
-
 import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.Set;
 
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.operators.MessageStream;
+import org.apache.samza.operators.MessageStreamImpl;
+import org.apache.samza.operators.functions.WatermarkFunction;
+
 /**
- * A stream operator specification that holds all the information required to transform 
+ * A stream operator specification that holds all the information required to transform
  * the input {@link org.apache.samza.operators.MessageStreamImpl} and produce the output
  * {@link org.apache.samza.operators.MessageStreamImpl}.
  *
diff --git a/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpecs.java b/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpecs.java
index 1b3b8aa63b..c752fe2057 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpecs.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/spec/OperatorSpecs.java
@@ -19,6 +19,10 @@
 
 package org.apache.samza.operators.spec;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.function.Function;
+
 import org.apache.samza.config.Config;
 import org.apache.samza.operators.KV;
 import org.apache.samza.operators.functions.FilterFunction;
@@ -26,15 +30,13 @@
 import org.apache.samza.operators.functions.JoinFunction;
 import org.apache.samza.operators.functions.MapFunction;
 import org.apache.samza.operators.functions.SinkFunction;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
 import org.apache.samza.operators.windows.internal.WindowInternal;
 import org.apache.samza.serializers.Serde;
 import org.apache.samza.system.StreamSpec;
+import org.apache.samza.table.TableSpec;
 import org.apache.samza.task.TaskContext;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.function.Function;
-
 
 /**
  * Factory methods for creating {@link OperatorSpec} instances.
@@ -242,4 +244,38 @@ public static <M> StreamOperatorSpec<M, M> createMergeOperatorSpec(String opId)
         },
         OperatorSpec.OpCode.MERGE, opId);
   }
+
+  /**
+   * Creates a {@link StreamTableJoinOperatorSpec} with a join function.
+   *
+   * @param tableSpec the table spec for the table on the right side of the join
+   * @param joinFn the user-defined join function to get join keys and results
+   * @param opId the unique ID of the operator
+   * @param <K> the type of join key
+   * @param <M> the type of input messages
+   * @param <R> the type of table record
+   * @param <JM> the type of the join result
+   * @return the {@link StreamTableJoinOperatorSpec}
+   */
+  public static <K, M, R, JM> StreamTableJoinOperatorSpec<K, M, R, JM> createStreamTableJoinOperatorSpec(
+      TableSpec tableSpec, StreamTableJoinFunction<K, M, R, JM> joinFn, String opId) {
+    return new StreamTableJoinOperatorSpec(tableSpec, joinFn, opId);
+  }
+
+  /**
+   * Creates a {@link SendToTableOperatorSpec} with a key extractor and a value extractor function,
+   * the type of incoming message is expected to be KV&#60;K, V&#62;.
+   *
+   * @param inputOpSpec the operator spec for the input stream
+   * @param tableSpec the table spec for the underlying table
+   * @param opId the unique ID of the operator
+   * @param <K> the type of the table record key
+   * @param <V> the type of the table record value
+   * @return the {@link SendToTableOperatorSpec}
+   */
+  public static <K, V> SendToTableOperatorSpec<K, V> createSendToTableOperatorSpec(
+      OperatorSpec<?, KV<K, V>> inputOpSpec, TableSpec tableSpec, String opId) {
+    return new SendToTableOperatorSpec(inputOpSpec, tableSpec, opId);
+  }
+
 }
diff --git a/samza-core/src/main/java/org/apache/samza/operators/spec/SendToTableOperatorSpec.java b/samza-core/src/main/java/org/apache/samza/operators/spec/SendToTableOperatorSpec.java
new file mode 100644
index 0000000000..9084be2fad
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/operators/spec/SendToTableOperatorSpec.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators.spec;
+
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.operators.KV;
+import org.apache.samza.operators.functions.WatermarkFunction;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * The spec for operator that writes a stream to a table by extracting keys and values
+ * from the incoming messages.
+ *
+ * @param <K> the type of the table record key
+ * @param <V> the type of the table record value
+ */
+@InterfaceStability.Unstable
+public class SendToTableOperatorSpec<K, V> extends OperatorSpec<KV<K, V>, Void> {
+
+  private final OperatorSpec<?, KV<K, V>> inputOpSpec;
+  private final TableSpec tableSpec;
+
+  /**
+   * Constructor for a {@link SendToTableOperatorSpec}.
+   *
+   * @param inputOpSpec  the operator spec of the input stream
+   * @param tableSpec  the table spec of the table written to
+   * @param opId  the unique ID for this operator
+   */
+  SendToTableOperatorSpec(OperatorSpec<?, KV<K, V>> inputOpSpec, TableSpec tableSpec, String opId) {
+    super(OpCode.SEND_TO, opId);
+    this.inputOpSpec = inputOpSpec;
+    this.tableSpec = tableSpec;
+  }
+
+  public OperatorSpec<?, KV<K, V>> getInputOpSpec() {
+    return inputOpSpec;
+  }
+
+  public TableSpec getTableSpec() {
+    return tableSpec;
+  }
+
+  @Override
+  public WatermarkFunction getWatermarkFn() {
+    return null;
+  }
+}
diff --git a/samza-core/src/main/java/org/apache/samza/operators/spec/StreamTableJoinOperatorSpec.java b/samza-core/src/main/java/org/apache/samza/operators/spec/StreamTableJoinOperatorSpec.java
new file mode 100644
index 0000000000..730913a3de
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/operators/spec/StreamTableJoinOperatorSpec.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators.spec;
+
+import org.apache.samza.annotation.InterfaceStability;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
+import org.apache.samza.operators.functions.WatermarkFunction;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * The spec for stream-table join operator that retrieves a record from the table using key
+ * derived from the incoming message and joins with the incoming message.
+ *
+ * @param <M>  the type of input messages
+ * @param <R>  the type of table record
+ * @param <JM>  the type of join result
+ */
+@InterfaceStability.Unstable
+public class StreamTableJoinOperatorSpec<K, M, R, JM> extends OperatorSpec<M, JM> {
+
+  private final TableSpec tableSpec;
+  private final StreamTableJoinFunction<K, M, R, JM> joinFn;
+
+  /**
+   * Constructor for {@link StreamTableJoinOperatorSpec}.
+   *
+   * @param tableSpec  the table spec for the table on the right side of the join
+   * @param joinFn  the user-defined join function to get join keys and results
+   * @param opId  the unique ID for this operator
+   */
+  StreamTableJoinOperatorSpec(TableSpec tableSpec, StreamTableJoinFunction<K, M, R, JM> joinFn, String opId) {
+    super(OpCode.JOIN, opId);
+    this.tableSpec = tableSpec;
+    this.joinFn = joinFn;
+  }
+
+  public TableSpec getTableSpec() {
+    return tableSpec;
+  }
+
+  public StreamTableJoinFunction<K, M, R, JM> getJoinFn() {
+    return this.joinFn;
+  }
+
+  @Override
+  public WatermarkFunction getWatermarkFn() {
+    return joinFn instanceof WatermarkFunction ? (WatermarkFunction) joinFn : null;
+  }
+
+}
diff --git a/samza-core/src/main/java/org/apache/samza/table/TableManager.java b/samza-core/src/main/java/org/apache/samza/table/TableManager.java
new file mode 100644
index 0000000000..c3555f325c
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/table/TableManager.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.serializers.Serde;
+import org.apache.samza.storage.StorageEngine;
+import org.apache.samza.util.Util;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A {@link TableManager} manages tables within a Samza task. For each table, it maintains
+ * the {@link TableSpec} and the {@link TableProvider}. It is used at execution for
+ * {@link org.apache.samza.container.TaskInstance} to retrieve table instances for
+ * read/write operations.
+ *
+ * A {@link TableManager} is constructed from job configuration, the {@link TableSpec}
+ * and {@link TableProvider} are constructed by processing the job configuration.
+ *
+ * After a {@link TableManager} is constructed, local tables are associated with
+ * local store instances created during {@link org.apache.samza.container.SamzaContainer}
+ * initialization.
+ *
+ * Method {@link TableManager#getTable(String)} will throw {@link IllegalStateException},
+ * if it's called before initialization.
+ *
+ * For store backed tables, the list of stores must be injected into the constructor.
+ */
+public class TableManager {
+
+  static public class TableCtx {
+    private TableSpec tableSpec;
+    private TableProvider tableProvider;
+  }
+
+  private final Logger logger = LoggerFactory.getLogger(TableManager.class.getName());
+
+  // tableId -> TableCtx
+  private final Map<String, TableCtx> tables = new HashMap<>();
+
+  private boolean localTablesInitialized;
+
+  /**
+   * Construct a table manager instance
+   * @param config the job configuration
+   * @param serdes Serde instances for tables
+   */
+  public TableManager(Config config, Map<String, Serde<Object>> serdes) {
+
+    new JavaTableConfig(config).getTableIds().forEach(tableId -> {
+
+        // Construct the table provider
+        String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId));
+
+        // Construct the KVSerde
+        JavaTableConfig tableConfig = new JavaTableConfig(config);
+        KVSerde serde = KVSerde.of(
+            serdes.get(tableConfig.getKeySerde(tableId)),
+            serdes.get(tableConfig.getValueSerde(tableId)));
+
+        TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory,
+            config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + "."));
+
+        addTable(tableSpec);
+
+        logger.info("Added table " + tableSpec.getId());
+      });
+  }
+
+  /**
+   * Initialize all local table
+   * @param stores stores created locally
+   */
+  public void initLocalTables(Map<String, StorageEngine> stores) {
+    tables.values().forEach(ctx -> {
+        if (ctx.tableProvider instanceof LocalStoreBackedTableProvider) {
+          StorageEngine store = stores.get(ctx.tableSpec.getId());
+          if (store == null) {
+            throw new SamzaException(String.format(
+                "Backing store for table %s was not injected by SamzaContainer",
+                ctx.tableSpec.getId()));
+          }
+          ((LocalStoreBackedTableProvider) ctx.tableProvider).init(store);
+        }
+      });
+
+    localTablesInitialized = true;
+  }
+
+  /**
+   * Add a table to the table manager
+   * @param tableSpec the table spec
+   */
+  private void addTable(TableSpec tableSpec) {
+    if (tables.containsKey(tableSpec.getId())) {
+      throw new SamzaException("Table " + tableSpec.getId() + " already exists");
+    }
+    TableCtx ctx = new TableCtx();
+    TableProviderFactory tableProviderFactory = Util.getObj(tableSpec.getTableProviderFactoryClassName());
+    ctx.tableProvider = tableProviderFactory.getTableProvider(tableSpec);
+    ctx.tableSpec = tableSpec;
+    tables.put(tableSpec.getId(), ctx);
+  }
+
+  /**
+   * Start the table manager, internally it starts all tables
+   */
+  public void start() {
+    tables.values().forEach(ctx -> ctx.tableProvider.start());
+  }
+
+  /**
+   * Shutdown the table manager, internally it shuts down all tables
+   */
+  public void shutdown() {
+    tables.values().forEach(ctx -> ctx.tableProvider.stop());
+  }
+
+  /**
+   * Get a table instance
+   * @param tableId Id of the table
+   * @return table instance
+   */
+  public Table getTable(String tableId) {
+    if (!localTablesInitialized) {
+      throw new IllegalStateException("Local tables in TableManager not initialized.");
+    }
+    return tables.get(tableId).tableProvider.getTable();
+  }
+}
diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
index 412e9dc5cd..f465bfce88 100644
--- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
+++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
@@ -20,65 +20,37 @@
 package org.apache.samza.container
 
 import java.io.File
+import java.net.{URL, UnknownHostException}
 import java.nio.file.Path
 import java.util
-import java.util.concurrent.{ExecutorService, Executors, TimeUnit}
-import java.net.{URL, UnknownHostException}
 import java.util.Base64
+import java.util.concurrent.{ExecutorService, Executors, TimeUnit}
 
-import org.apache.samza.{SamzaContainerStatus, SamzaException}
 import org.apache.samza.checkpoint.{CheckpointListener, CheckpointManagerFactory, OffsetManager, OffsetManagerMetrics}
 import org.apache.samza.config.JobConfig.Config2Job
 import org.apache.samza.config.MetricsConfig.Config2Metrics
 import org.apache.samza.config.SerializerConfig.Config2Serializer
-import org.apache.samza.config._
 import org.apache.samza.config.StorageConfig.Config2Storage
 import org.apache.samza.config.StreamConfig.Config2Stream
 import org.apache.samza.config.SystemConfig.Config2System
 import org.apache.samza.config.TaskConfig.Config2Task
-import org.apache.samza.container.disk.DiskQuotaPolicyFactory
-import org.apache.samza.container.disk.DiskSpaceMonitor
+import org.apache.samza.config._
 import org.apache.samza.container.disk.DiskSpaceMonitor.Listener
-import org.apache.samza.container.disk.NoThrottlingDiskQuotaPolicyFactory
-import org.apache.samza.container.disk.PollingScanDiskSpaceMonitor
+import org.apache.samza.container.disk.{DiskQuotaPolicyFactory, DiskSpaceMonitor, NoThrottlingDiskQuotaPolicyFactory, PollingScanDiskSpaceMonitor}
 import org.apache.samza.container.host.{StatisticsMonitorImpl, SystemMemoryStatistics, SystemStatisticsMonitor}
 import org.apache.samza.coordinator.stream.CoordinatorStreamSystemFactory
-import org.apache.samza.job.model.ContainerModel
 import org.apache.samza.job.model.JobModel
-import org.apache.samza.metrics.JmxServer
-import org.apache.samza.metrics.JvmMetrics
-import org.apache.samza.metrics.MetricsRegistryMap
-import org.apache.samza.metrics.MetricsReporter
-import org.apache.samza.serializers.IntermediateMessageSerde
-import org.apache.samza.serializers.NoOpSerde
-import org.apache.samza.serializers.SerializableSerde
-import org.apache.samza.serializers.Serde
-import org.apache.samza.serializers.SerdeFactory
-import org.apache.samza.serializers.SerdeManager
-import org.apache.samza.serializers.StringSerde
+import org.apache.samza.metrics.{JmxServer, JvmMetrics, MetricsRegistryMap, MetricsReporter}
+import org.apache.samza.serializers._
 import org.apache.samza.serializers.model.SamzaObjectMapper
-import org.apache.samza.storage.StorageEngineFactory
-import org.apache.samza.storage.TaskStorageManager
-import org.apache.samza.system.StreamMetadataCache
-import org.apache.samza.system.SystemConsumers
-import org.apache.samza.system.SystemConsumersMetrics
-import org.apache.samza.system.SystemFactory
-import org.apache.samza.system.SystemProducers
-import org.apache.samza.system.SystemProducersMetrics
-import org.apache.samza.system.SystemStream
-import org.apache.samza.system.SystemStreamPartition
-import org.apache.samza.system.chooser.DefaultChooser
-import org.apache.samza.system.chooser.MessageChooserFactory
-import org.apache.samza.system.chooser.RoundRobinChooserFactory
+import org.apache.samza.storage.{StorageEngineFactory, TaskStorageManager}
+import org.apache.samza.system._
+import org.apache.samza.system.chooser.{DefaultChooser, MessageChooserFactory, RoundRobinChooserFactory}
+import org.apache.samza.table.TableManager
 import org.apache.samza.task._
-import org.apache.samza.util.HighResolutionClock
-import org.apache.samza.util.ExponentialSleepStrategy
-import org.apache.samza.util.Logging
-import org.apache.samza.util.Throttleable
-import org.apache.samza.util.MetricsReporterLoader
-import org.apache.samza.util.SystemClock
-import org.apache.samza.util.Util
 import org.apache.samza.util.Util.asScalaClock
+import org.apache.samza.util._
+import org.apache.samza.{SamzaContainerStatus, SamzaException}
 
 import scala.collection.JavaConverters._
 
@@ -568,6 +540,11 @@ object SamzaContainer extends Logging {
         new StorageConfig(config).getChangeLogDeleteRetentionsInMs,
         new SystemClock)
 
+      val tableManager = new TableManager(config, serdes.asJava)
+      tableManager.initLocalTables(taskStores.asJava)
+
+      info("Got table manager");
+
       val systemStreamPartitions = taskModel
         .getSystemStreamPartitions
         .asScala
@@ -586,6 +563,7 @@ object SamzaContainer extends Logging {
           containerContext = containerContext,
           offsetManager = offsetManager,
           storageManager = storageManager,
+          tableManager = tableManager,
           reporters = reporters,
           systemStreamPartitions = systemStreamPartitions,
           exceptionHandler = TaskInstanceExceptionHandler(taskInstanceMetrics, config),
@@ -711,6 +689,7 @@ class SamzaContainer(
       startOffsetManager
       startLocalityManager
       startStores
+      startTableManager
       startDiskSpaceMonitor
       startHostStatisticsMonitor
       startProducers
@@ -745,6 +724,7 @@ class SamzaContainer(
 
       shutdownConsumers
       shutdownTask
+      shutdownTableManager
       shutdownStores
       shutdownDiskSpaceMonitor
       shutdownHostStatisticsMonitor
@@ -885,9 +865,9 @@ class SamzaContainer(
   }
 
   def startStores {
-    info("Starting task instance stores.")
     taskInstances.values.foreach(taskInstance => {
       val startTime = System.currentTimeMillis()
+      info("Starting stores in task instance %s" format taskInstance.taskName)
       taskInstance.startStores
       // Measuring the time to restore the stores
       val timeToRestore = System.currentTimeMillis() - startTime
@@ -898,6 +878,13 @@ class SamzaContainer(
     })
   }
 
+  def startTableManager: Unit = {
+    taskInstances.values.foreach(taskInstance => {
+      info("Starting table manager in task instance %s" format taskInstance.taskName)
+      taskInstance.startTableManager
+    })
+  }
+
   def startTask {
     info("Initializing stream tasks.")
 
@@ -1003,6 +990,12 @@ class SamzaContainer(
     taskInstances.values.foreach(_.shutdownStores)
   }
 
+  def shutdownTableManager: Unit = {
+    info("Shutting down task instance table manager.")
+
+    taskInstances.values.foreach(_.shutdownTableManager)
+  }
+
   def shutdownLocalityManager {
     if(localityManager != null) {
       info("Shutting down locality manager.")
diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala
index acec365deb..f2a50743e7 100644
--- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala
+++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala
@@ -27,20 +27,9 @@ import org.apache.samza.config.StreamConfig.Config2Stream
 import org.apache.samza.job.model.JobModel
 import org.apache.samza.metrics.MetricsReporter
 import org.apache.samza.storage.TaskStorageManager
-import org.apache.samza.system.IncomingMessageEnvelope
-import org.apache.samza.system.StreamMetadataCache
-import org.apache.samza.system.SystemAdmin
-import org.apache.samza.system.SystemConsumers
-import org.apache.samza.system.SystemStreamPartition
-import org.apache.samza.task.AsyncStreamTask
-import org.apache.samza.task.ClosableTask
-import org.apache.samza.task.EndOfStreamListenerTask
-import org.apache.samza.task.InitableTask
-import org.apache.samza.task.ReadableCoordinator
-import org.apache.samza.task.StreamTask
-import org.apache.samza.task.TaskCallbackFactory
-import org.apache.samza.task.TaskInstanceCollector
-import org.apache.samza.task.WindowableTask
+import org.apache.samza.system._
+import org.apache.samza.table.TableManager
+import org.apache.samza.task._
 import org.apache.samza.util.Logging
 
 import scala.collection.JavaConverters._
@@ -56,6 +45,7 @@ class TaskInstance(
   containerContext: SamzaContainerContext,
   val offsetManager: OffsetManager = new OffsetManager,
   storageManager: TaskStorageManager = null,
+  tableManager: TableManager = null,
   reporters: Map[String, MetricsReporter] = Map(),
   val systemStreamPartitions: Set[SystemStreamPartition] = Set(),
   val exceptionHandler: TaskInstanceExceptionHandler = new TaskInstanceExceptionHandler,
@@ -68,7 +58,7 @@ class TaskInstance(
   val isAsyncTask = task.isInstanceOf[AsyncStreamTask]
 
   val context = new TaskContextImpl(taskName,metrics, containerContext, systemStreamPartitions.asJava, offsetManager,
-                                    storageManager, jobModel, streamMetadataCache)
+                                    storageManager, tableManager, jobModel, streamMetadataCache)
 
   // store the (ssp -> if this ssp is catched up) mapping. "catched up"
   // means the same ssp in other taskInstances have the same offset as
@@ -101,6 +91,16 @@ class TaskInstance(
     }
   }
 
+  def startTableManager {
+    if (tableManager != null) {
+      debug("Starting table manager for taskName: %s" format taskName)
+
+      tableManager.start
+    } else {
+      debug("Skipping table manager initialization for taskName: %s" format taskName)
+    }
+  }
+
   def initTask {
     if (isInitableTask) {
       debug("Initializing task for taskName: %s" format taskName)
@@ -225,6 +225,16 @@ class TaskInstance(
     }
   }
 
+  def shutdownTableManager {
+    if (tableManager != null) {
+      debug("Shutting down table manager for taskName: %s" format taskName)
+
+      tableManager.shutdown
+    } else {
+      debug("Skipping table manager shutdown for taskName: %s" format taskName)
+    }
+  }
+
   override def toString() = "TaskInstance for class %s and taskName %s." format (task.getClass.getName, taskName)
 
   def toDetailedString() = "TaskInstance [taskName = %s, windowable=%s, closable=%s endofstreamlistener=%s]" format
diff --git a/samza-core/src/test/java/org/apache/samza/config/TestJavaTableConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestJavaTableConfig.java
new file mode 100644
index 0000000000..2775ca77bf
--- /dev/null
+++ b/samza-core/src/test/java/org/apache/samza/config/TestJavaTableConfig.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.config;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+
+import com.google.common.collect.Sets;
+
+import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+
+public class TestJavaTableConfig {
+  @Test
+  public void testGetTableIds() {
+    Set<String> ids = Sets.newHashSet("t1", "t2");
+    Map<String, String> map = ids.stream()
+        .map(id -> String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, id))
+        .collect(Collectors.toMap(key -> key, key -> key + "-provider-factory"));
+    JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map));
+
+    assertEquals(2, tableConfig.getTableIds().size());
+
+    ids.removeAll(tableConfig.getTableIds());
+    assertTrue(ids.isEmpty());
+  }
+
+  @Test
+  public void testGetTableProperties() {
+    Map<String, String> map = new HashMap<>();
+    map.put("tables.t1.spec", "t1-spec");
+    map.put("tables.t1.provider.factory", "t1-provider-factory");
+    JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(map));
+    assertEquals("t1-provider-factory", tableConfig.getTableProviderFactory("t1"));
+  }
+
+}
diff --git a/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java b/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java
index d97d4945fe..96e234e604 100644
--- a/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java
+++ b/samza-core/src/test/java/org/apache/samza/operators/TestMessageStreamImpl.java
@@ -18,7 +18,12 @@
  */
 package org.apache.samza.operators;
 
-import com.google.common.collect.ImmutableList;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
 import org.apache.samza.operators.data.TestMessageEnvelope;
 import org.apache.samza.operators.data.TestOutputMessageEnvelope;
 import org.apache.samza.operators.functions.FilterFunction;
@@ -27,14 +32,17 @@
 import org.apache.samza.operators.functions.JoinFunction;
 import org.apache.samza.operators.functions.MapFunction;
 import org.apache.samza.operators.functions.SinkFunction;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
 import org.apache.samza.operators.spec.JoinOperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec.OpCode;
 import org.apache.samza.operators.spec.OutputOperatorSpec;
 import org.apache.samza.operators.spec.OutputStreamImpl;
 import org.apache.samza.operators.spec.PartitionByOperatorSpec;
+import org.apache.samza.operators.spec.SendToTableOperatorSpec;
 import org.apache.samza.operators.spec.SinkOperatorSpec;
 import org.apache.samza.operators.spec.StreamOperatorSpec;
+import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
 import org.apache.samza.operators.spec.WindowOperatorSpec;
 import org.apache.samza.operators.stream.IntermediateMessageStreamImpl;
 import org.apache.samza.operators.windows.Window;
@@ -42,14 +50,11 @@
 import org.apache.samza.operators.windows.Windows;
 import org.apache.samza.serializers.KVSerde;
 import org.apache.samza.serializers.Serde;
+import org.apache.samza.table.TableSpec;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
-import java.time.Duration;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.function.Function;
-import java.util.function.Supplier;
+import com.google.common.collect.ImmutableList;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -312,6 +317,57 @@ public void testJoin() {
     assertEquals(rightInputOpSpec, ((JoinOperatorSpec) leftRegisteredOpSpec).getRightInputOpSpec());
   }
 
+  @Test
+  public void testSendToTable() {
+    StreamGraphImpl mockGraph = mock(StreamGraphImpl.class);
+    OperatorSpec inputOpSpec = mock(OperatorSpec.class);
+    MessageStreamImpl<TestMessageEnvelope> source = new MessageStreamImpl<>(mockGraph, inputOpSpec);
+
+    TableSpec tableSpec = new TableSpec();
+    TableImpl table = new TableImpl(tableSpec);
+
+    source.sendTo(table);
+
+    ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
+    verify(inputOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
+    OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();
+
+    assertTrue(registeredOpSpec instanceof SendToTableOperatorSpec);
+    SendToTableOperatorSpec sendToTableOperatorSpec = (SendToTableOperatorSpec) registeredOpSpec;
+
+    assertEquals(OpCode.SEND_TO, sendToTableOperatorSpec.getOpCode());
+    assertEquals(inputOpSpec, sendToTableOperatorSpec.getInputOpSpec());
+    assertEquals(tableSpec, sendToTableOperatorSpec.getTableSpec());
+  }
+
+  @Test
+  public void testStreamTableJoin() {
+    StreamGraphImpl mockGraph = mock(StreamGraphImpl.class);
+    OperatorSpec leftInputOpSpec = mock(OperatorSpec.class);
+    MessageStreamImpl<KV<String, TestMessageEnvelope>> source1 = new MessageStreamImpl<>(mockGraph, leftInputOpSpec);
+    OperatorSpec rightInputOpSpec = mock(OperatorSpec.class);
+    MessageStreamImpl<TestMessageEnvelope> source2 = new MessageStreamImpl<>(mockGraph, rightInputOpSpec);
+
+    TableSpec tableSpec = new TableSpec();
+    TableImpl table = new TableImpl(tableSpec);
+
+    source2.sendTo(table);
+
+    StreamTableJoinFunction<String, KV<String, TestMessageEnvelope>, KV<String, TestMessageEnvelope>, TestOutputMessageEnvelope>
+        mockJoinFn = mock(StreamTableJoinFunction.class);
+    source1.join(table, mockJoinFn);
+
+    ArgumentCaptor<OperatorSpec> leftRegisteredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
+    verify(leftInputOpSpec).registerNextOperatorSpec(leftRegisteredOpCaptor.capture());
+    OperatorSpec<?, TestMessageEnvelope> leftRegisteredOpSpec = leftRegisteredOpCaptor.getValue();
+
+    assertTrue(leftRegisteredOpSpec instanceof StreamTableJoinOperatorSpec);
+    StreamTableJoinOperatorSpec joinOpSpec = (StreamTableJoinOperatorSpec) leftRegisteredOpSpec;
+    assertEquals(OpCode.JOIN, joinOpSpec.getOpCode());
+    assertEquals(mockJoinFn, joinOpSpec.getJoinFn());
+    assertEquals(tableSpec, joinOpSpec.getTableSpec());
+  }
+
   @Test
   public void testMerge() {
     StreamGraphImpl mockGraph = mock(StreamGraphImpl.class);
diff --git a/samza-core/src/test/java/org/apache/samza/operators/TestStreamGraphImpl.java b/samza-core/src/test/java/org/apache/samza/operators/TestStreamGraphImpl.java
index cf0a198cdd..3bb44b55d7 100644
--- a/samza-core/src/test/java/org/apache/samza/operators/TestStreamGraphImpl.java
+++ b/samza-core/src/test/java/org/apache/samza/operators/TestStreamGraphImpl.java
@@ -12,14 +12,16 @@
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
+ * KIND, either express or implied.  See the License for THE
  * specific language governing permissions and limitations
  * under the License.
  */
 package org.apache.samza.operators;
 
-import com.google.common.collect.ImmutableList;
-import junit.framework.Assert;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
 import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
 import org.apache.samza.config.JobConfig;
@@ -33,10 +35,11 @@
 import org.apache.samza.serializers.NoOpSerde;
 import org.apache.samza.serializers.Serde;
 import org.apache.samza.system.StreamSpec;
+import org.apache.samza.table.TableSpec;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.List;
+import com.google.common.collect.ImmutableList;
+import junit.framework.Assert;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -583,4 +586,16 @@ public void testGetInputStreamPreservesInsertionOrder() {
     Assert.assertEquals(inputSpecs.get(1).getStreamSpec(), testStreamSpec2);
     Assert.assertEquals(inputSpecs.get(2).getStreamSpec(), testStreamSpec3);
   }
+
+  @Test
+  public void testGetTable() {
+    ApplicationRunner mockRunner = mock(ApplicationRunner.class);
+    Config mockConfig = mock(Config.class);
+    StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
+
+    BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class);
+    when(mockTableDescriptor.getTableSpec()).thenReturn(
+        new TableSpec("t1", KVSerde.of(new NoOpSerde(), new NoOpSerde()), "", new HashMap<>()));
+    Assert.assertNotNull(graph.getTable(mockTableDescriptor));
+  }
 }
diff --git a/samza-core/src/test/java/org/apache/samza/operators/impl/TestStreamTableJoinOperatorImpl.java b/samza-core/src/test/java/org/apache/samza/operators/impl/TestStreamTableJoinOperatorImpl.java
new file mode 100644
index 0000000000..d8b2e8d868
--- /dev/null
+++ b/samza-core/src/test/java/org/apache/samza/operators/impl/TestStreamTableJoinOperatorImpl.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.operators.impl;
+
+import java.util.Collection;
+
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
+import org.apache.samza.operators.KV;
+import org.apache.samza.operators.data.TestMessageEnvelope;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
+import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
+import org.apache.samza.table.ReadableTable;
+import org.apache.samza.table.TableSpec;
+import org.apache.samza.task.MessageCollector;
+import org.apache.samza.task.TaskContext;
+import org.apache.samza.task.TaskCoordinator;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class TestStreamTableJoinOperatorImpl {
+  @Test
+  public void testHandleMessage() {
+
+    String tableId = "t1";
+    TableSpec tableSpec = mock(TableSpec.class);
+    when(tableSpec.getId()).thenReturn(tableId);
+
+    StreamTableJoinOperatorSpec mockJoinOpSpec = mock(StreamTableJoinOperatorSpec.class);
+    when(mockJoinOpSpec.getTableSpec()).thenReturn(tableSpec);
+    when(mockJoinOpSpec.getJoinFn()).thenReturn(
+        new StreamTableJoinFunction<String, KV<String, String>, KV<String, String>, String>() {
+          @Override
+          public String apply(KV<String, String> message, KV<String, String> record) {
+            if ("1".equals(message.getKey())) {
+              Assert.assertEquals("m1", message.getValue());
+              Assert.assertEquals("r1", record.getValue());
+              return "m1r1";
+            } else if ("2".equals(message.getKey())) {
+              Assert.assertEquals("m2", message.getValue());
+              Assert.assertNull(record);
+              return null;
+            }
+            throw new SamzaException("Should never reach here!");
+          }
+
+          @Override
+          public String getMessageKey(KV<String, String> message) {
+            return message.getKey();
+          }
+
+          @Override
+          public String getRecordKey(KV<String, String> record) {
+            return record.getKey();
+          }
+        });
+    Config config = mock(Config.class);
+    ReadableTable table = mock(ReadableTable.class);
+    when(table.get("1")).thenReturn("r1");
+    when(table.get("2")).thenReturn(null);
+    TaskContext mockTaskContext = mock(TaskContext.class);
+    when(mockTaskContext.getTable(tableId)).thenReturn(table);
+
+    MessageCollector mockMessageCollector = mock(MessageCollector.class);
+    TaskCoordinator mockTaskCoordinator = mock(TaskCoordinator.class);
+
+    StreamTableJoinOperatorImpl streamTableJoinOperator = new StreamTableJoinOperatorImpl(
+        mockJoinOpSpec, config, mockTaskContext);
+
+    // Table has the key
+    Collection<TestMessageEnvelope> result;
+    result = streamTableJoinOperator.handleMessage(KV.of("1", "m1"), mockMessageCollector, mockTaskCoordinator);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals("m1r1", result.iterator().next());
+    // Table doesn't have the key
+    result = streamTableJoinOperator.handleMessage(KV.of("2", "m2"), mockMessageCollector, mockTaskCoordinator);
+    Assert.assertEquals(0, result.size());
+  }
+
+}
diff --git a/samza-core/src/test/java/org/apache/samza/table/TestTableManager.java b/samza-core/src/test/java/org/apache/samza/table/TestTableManager.java
new file mode 100644
index 0000000000..df5b9e5746
--- /dev/null
+++ b/samza-core/src/test/java/org/apache/samza/table/TestTableManager.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.table;
+
+import java.lang.reflect.Field;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.MapConfig;
+import org.apache.samza.config.SerializerConfig;
+import org.apache.samza.serializers.IntegerSerde;
+import org.apache.samza.serializers.Serde;
+import org.apache.samza.serializers.SerializableSerde;
+import org.apache.samza.serializers.StringSerde;
+import org.apache.samza.storage.StorageEngine;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class TestTableManager {
+
+  private static final String TABLE_ID = "t1";
+
+  public static class DummyTableProviderFactory implements TableProviderFactory {
+
+    static Table table;
+    static LocalStoreBackedTableProvider tableProvider;
+
+    @Override
+    public TableProvider getTableProvider(TableSpec tableSpec) {
+      table = mock(Table.class);
+      tableProvider = mock(LocalStoreBackedTableProvider.class);
+      when(tableProvider.getTable()).thenReturn(table);
+      return tableProvider;
+    }
+  }
+
+  @Test
+  public void testInitByConfig() {
+    Map<String, String> map = new HashMap<>();
+    map.put(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, TABLE_ID), DummyTableProviderFactory.class.getName());
+    map.put(String.format("tables.%s.some.config", TABLE_ID), "xyz");
+    addKeySerde(map);
+    addValueSerde(map);
+    doTestInit(map);
+  }
+
+  @Test(expected = Exception.class)
+  public void testInitFailsWithoutProviderFactory() {
+    Map<String, String> map = new HashMap<>();
+    addKeySerde(map);
+    addValueSerde(map);
+    doTestInit(map);
+  }
+
+  @Test(expected = Exception.class)
+  public void testInitFailsWithoutKeySerde() {
+    Map<String, String> map = new HashMap<>();
+    map.put(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, TABLE_ID), DummyTableProviderFactory.class.getName());
+    addValueSerde(map);
+    doTestInit(map);
+  }
+
+  @Test(expected = Exception.class)
+  public void testInitFailsWithoutValueSerde() {
+    Map<String, String> map = new HashMap<>();
+    map.put(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, TABLE_ID), DummyTableProviderFactory.class.getName());
+    addValueSerde(map);
+    doTestInit(map);
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void testInitFailsWithoutInitializingLocalTables() {
+    TableManager tableManager = new TableManager(new MapConfig(new HashMap<>()), new HashMap<>());
+    tableManager.getTable("dummy");
+  }
+
+  private void doTestInit(Map<String, String> map) {
+    Map<String, StorageEngine> storageEngines = new HashMap<>();
+    storageEngines.put(TABLE_ID, mock(StorageEngine.class));
+
+    Map<String, Serde<Object>> serdeMap = new HashMap<>();
+    SerializableSerde<Serde> serializableSerde = new SerializableSerde();
+    map.keySet().stream()
+        .filter(k -> k.endsWith(SerializerConfig.SERIALIZED_INSTANCE_SUFFIX()))
+        .forEach(k -> {
+            String serdeName = k
+                .replace(String.format(SerializerConfig.SERIALIZER_PREFIX(), ""), "")
+                .replace(SerializerConfig.SERIALIZED_INSTANCE_SUFFIX(), "");
+            String serializedSerde = map.get(k);
+            byte[] bytes = Base64.getDecoder().decode(serializedSerde);
+            Serde serde = serializableSerde.fromBytes(bytes);
+            serdeMap.put(serdeName, serde);
+          });
+
+    TableManager tableManager = new TableManager(new MapConfig(map), serdeMap);
+    tableManager.initLocalTables(storageEngines);
+
+    Table table = tableManager.getTable(TABLE_ID);
+    verify(DummyTableProviderFactory.tableProvider, times(1)).init(anyObject());
+    Assert.assertEquals(DummyTableProviderFactory.table, table);
+
+    Map<String, TableManager.TableCtx> ctxMap = getFieldValue(tableManager, "tables");
+    TableManager.TableCtx ctx = ctxMap.get(TABLE_ID);
+
+    TableSpec tableSpec = getFieldValue(ctx, "tableSpec");
+    Assert.assertEquals(TABLE_ID, tableSpec.getId());
+    Assert.assertEquals(DummyTableProviderFactory.class.getName(), tableSpec.getTableProviderFactoryClassName());
+    Assert.assertEquals(IntegerSerde.class, tableSpec.getSerde().getKeySerde().getClass());
+    Assert.assertEquals(StringSerde.class, tableSpec.getSerde().getValueSerde().getClass());
+    Assert.assertEquals("xyz", tableSpec.getConfig().get("some.config"));
+
+    TableProvider tableProvider = getFieldValue(ctx, "tableProvider");
+    Assert.assertNotNull(tableProvider);
+  }
+
+  private void addKeySerde(Map<String, String> map) {
+    String serdeId = "key-serde";
+    map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId),
+        serializeSerde(new IntegerSerde()));
+    map.put(String.format(JavaTableConfig.TABLE_KEY_SERDE, TABLE_ID), serdeId);
+  }
+
+  private void addValueSerde(Map<String, String> map) {
+    String serdeId = "value-serde";
+    map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId),
+            serializeSerde(new StringSerde("UTF-8")));
+    map.put(String.format(JavaTableConfig.TABLE_VALUE_SERDE, TABLE_ID), serdeId);
+  }
+
+  private String serializeSerde(Serde serde) {
+    return Base64.getEncoder().encodeToString(new SerializableSerde().toBytes(serde));
+  }
+
+  private <T> T getFieldValue(Object object, String fieldName) {
+    Field field = null;
+    try {
+      field = object.getClass().getDeclaredField(fieldName);
+      field.setAccessible(true);
+      return (T) field.get(object);
+    } catch (NoSuchFieldException | IllegalAccessException ex) {
+      throw new SamzaException(ex);
+    } finally {
+      if (field != null) {
+        field.setAccessible(false);
+      }
+    }
+  }
+
+}
diff --git a/samza-core/src/test/java/org/apache/samza/task/TestAsyncRunLoop.java b/samza-core/src/test/java/org/apache/samza/task/TestAsyncRunLoop.java
index b399f5fae2..28a4f8b513 100644
--- a/samza-core/src/test/java/org/apache/samza/task/TestAsyncRunLoop.java
+++ b/samza-core/src/test/java/org/apache/samza/task/TestAsyncRunLoop.java
@@ -30,6 +30,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
+
 import org.apache.samza.Partition;
 import org.apache.samza.checkpoint.OffsetManager;
 import org.apache.samza.config.Config;
@@ -47,16 +48,15 @@
 import org.apache.samza.system.TestSystemConsumers;
 import org.junit.Before;
 import org.junit.Test;
+
 import scala.Option;
 import scala.collection.JavaConverters;
 
 import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.*;
-import static org.mockito.Mockito.atLeastOnce;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.*;
 
 // TODO(spvenkat) SAMZA-1183: Fix all commented out tests.
 public class TestAsyncRunLoop {
@@ -86,7 +86,7 @@ TaskInstance createTaskInstance(AsyncStreamTask task, TaskName taskName, SystemS
     scala.collection.immutable.Set<SystemStreamPartition> sspSet = JavaConverters.asScalaSetConverter(Collections.singleton(ssp)).asScala().toSet();
     return new TaskInstance(task, taskName, mock(Config.class), taskInstanceMetrics,
         null, consumers, mock(TaskInstanceCollector.class), mock(SamzaContainerContext.class),
-        manager, null, null, sspSet, new TaskInstanceExceptionHandler(taskInstanceMetrics, new scala.collection.immutable.HashSet<String>()), null, null);
+        manager, null, null, null, sspSet, new TaskInstanceExceptionHandler(taskInstanceMetrics, new scala.collection.immutable.HashSet<String>()), null, null);
   }
 
   TaskInstance createTaskInstance(AsyncStreamTask task, TaskName taskName, SystemStreamPartition ssp) {
diff --git a/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableDescriptor.java b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableDescriptor.java
new file mode 100644
index 0000000000..2681fb336f
--- /dev/null
+++ b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv.inmemory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.storage.kv.BaseLocalStoreBackedTableDescriptor;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * Table descriptor for in-memory tables
+ *
+ * @param <K> the type of the key
+ * @param <V> the type of the value
+ */
+public class InMemoryTableDescriptor<K, V> extends BaseLocalStoreBackedTableDescriptor<K, V, InMemoryTableDescriptor<K, V>> {
+
+  public InMemoryTableDescriptor(String tableId) {
+    super(tableId);
+  }
+
+  @Override
+  protected void generateTableSpecConfig(Map<String, String> tableSpecConfig) {
+    super.generateTableSpecConfig(tableSpecConfig);
+  }
+
+  @Override
+  public TableSpec getTableSpec() {
+
+    validate();
+
+    Map<String, String> tableSpecConfig = new HashMap<>();
+    generateTableSpecConfig(tableSpecConfig);
+
+    return new TableSpec(tableId, serde, InMemoryTableProviderFactory.class.getName(), tableSpecConfig);
+  }
+
+  private void addInMemoryConfig(Map<String, String> map, String key, String value) {
+    map.put("inmemory." + key, value);
+  }
+}
diff --git a/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProvider.java b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProvider.java
new file mode 100644
index 0000000000..c1c2f1ca5c
--- /dev/null
+++ b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv.inmemory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.StorageConfig;
+import org.apache.samza.storage.kv.BaseLocalStoreBackedTableProvider;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * Table provider of an in-memory table
+ */
+public class InMemoryTableProvider extends BaseLocalStoreBackedTableProvider {
+
+  public InMemoryTableProvider(TableSpec tableSpec) {
+    super(tableSpec);
+  }
+
+  @Override
+  public Map<String, String> generateConfig(Map<String, String> config) {
+
+    Map<String, String> tableConfig = new HashMap<>();
+
+    // Store factory configuration
+    tableConfig.put(String.format(
+        StorageConfig.FACTORY(), tableSpec.getId()),
+        InMemoryKeyValueStorageEngineFactory.class.getName());
+
+    // Common store configuration
+    tableConfig.putAll(generateCommonStoreConfig(config));
+
+    // Rest of the configuration
+    tableSpec.getConfig().forEach((k, v) -> {
+      String realKey = k.startsWith("inmemory.") ?
+          String.format("stores.%s", tableSpec.getId()) + "." + k.substring("inmemory.".length())
+        : String.format(JavaTableConfig.TABLE_ID_PREFIX, tableSpec.getId()) + "." + k;
+      tableConfig.put(realKey, v);
+    });
+
+    logger.info("Generated configuration for table " + tableSpec.getId());
+
+    return tableConfig;
+  }
+
+}
diff --git a/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProviderFactory.java b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProviderFactory.java
new file mode 100644
index 0000000000..f05982a8f8
--- /dev/null
+++ b/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableProviderFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv.inmemory;
+
+import org.apache.samza.table.TableProvider;
+import org.apache.samza.table.TableProviderFactory;
+import org.apache.samza.table.TableSpec;
+
+/**
+ * Factory class for an in-memory table provider
+ */
+public class InMemoryTableProviderFactory implements TableProviderFactory {
+  @Override
+  public TableProvider getTableProvider(TableSpec tableSpec) {
+    return new InMemoryTableProvider(tableSpec);
+  }
+}
diff --git a/samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableDescriptor.java b/samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableDescriptor.java
new file mode 100644
index 0000000000..840fb70243
--- /dev/null
+++ b/samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableDescriptor.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.storage.kv.inmemory;
+
+import org.apache.samza.serializers.IntegerSerde;
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.serializers.StringSerde;
+import org.apache.samza.table.TableSpec;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestInMemoryTableDescriptor {
+  @Test
+  public void testTableSpec() {
+
+    TableSpec tableSpec = new InMemoryTableDescriptor<Integer, String>("1")
+        .withSerde(KVSerde.of(new IntegerSerde(), new StringSerde()))
+        .withConfig("inmemory.abc", "xyz")
+        .getTableSpec();
+
+    Assert.assertNotNull(tableSpec.getSerde());
+    Assert.assertNotNull(tableSpec.getSerde().getKeySerde());
+    Assert.assertNotNull(tableSpec.getSerde().getValueSerde());
+    Assert.assertEquals("xyz", getConfig(tableSpec, "abc"));
+  }
+
+  private String getConfig(TableSpec tableSpec, String key) {
+    return tableSpec.getConfig().get("inmemory." + key);
+  }
+}
diff --git a/samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableProvider.java b/samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableProvider.java
new file mode 100644
index 0000000000..76b7a666ba
--- /dev/null
+++ b/samza-kv-inmemory/src/test/java/org/apache/samza/storage/kv/inmemory/TestInMemoryTableProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.storage.kv.inmemory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.StorageConfig;
+import org.apache.samza.serializers.IntegerSerde;
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.table.TableProvider;
+import org.apache.samza.table.TableSpec;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+
+public class TestInMemoryTableProvider {
+  @Test
+  public void testGenerateConfig() {
+    Map<String, String> tableSpecConfig = new HashMap<>();
+    tableSpecConfig.put("inmemory.c1", "c1-value");
+    tableSpecConfig.put("inmemory.c2", "c2-value");
+    tableSpecConfig.put("c3", "c3-value");
+    tableSpecConfig.put("c4", "c4-value");
+
+    TableSpec tableSpec = new TableSpec("t1", KVSerde.of(new IntegerSerde(), new IntegerSerde()),
+        "my-table-provider-factory", tableSpecConfig);
+
+    Map<String, String> config = new HashMap<>();
+    config.put(String.format(JavaTableConfig.TABLE_KEY_SERDE, "t1"), "ks1");
+    config.put(String.format(JavaTableConfig.TABLE_VALUE_SERDE, "t1"), "vs1");
+
+    TableProvider tableProvider = new InMemoryTableProvider(tableSpec);
+    Map<String, String> tableConfig = tableProvider.generateConfig(config);
+
+    Assert.assertEquals("ks1", tableConfig.get(String.format(StorageConfig.KEY_SERDE(), "t1")));
+    Assert.assertEquals("vs1", tableConfig.get(String.format(StorageConfig.MSG_SERDE(), "t1")));
+    Assert.assertEquals(
+        InMemoryKeyValueStorageEngineFactory.class.getName(),
+        tableConfig.get(String.format(StorageConfig.FACTORY(), "t1")));
+    Assert.assertEquals("c1-value", tableConfig.get("stores.t1.c1"));
+    Assert.assertEquals("c2-value", tableConfig.get("stores.t1.c2"));
+    Assert.assertEquals("c3-value", tableConfig.get("tables.t1.c3"));
+    Assert.assertEquals("c4-value", tableConfig.get("tables.t1.c4"));
+  }
+}
diff --git a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableDescriptor.java b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableDescriptor.java
new file mode 100644
index 0000000000..2c6215943f
--- /dev/null
+++ b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableDescriptor.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * Table descriptor for RocksDb backed tables
+ *
+ * @param <K> the type of the key
+ * @param <V> the type of the value
+ */
+public class RocksDbTableDescriptor<K, V> extends BaseLocalStoreBackedTableDescriptor<K, V, RocksDbTableDescriptor<K, V>> {
+
+  static final public String WRITE_BATCH_SIZE = "write.batch.size";
+  static final public String OBJECT_CACHE_SIZE = "object.cache.size";
+  static final public String CONTAINER_CACHE_SIZE_BYTES = "container.cache.size.bytes";
+  static final public String CONTAINER_WRITE_BUFFER_SIZE_BYTES = "container.write.buffer.size.bytes";
+  static final public String ROCKSDB_COMPRESSION = "rocksdb.compression";
+  static final public String ROCKSDB_BLOCK_SIZE_BYTES = "rocksdb.block.size.bytes";
+  static final public String ROCKSDB_TTL_MS = "rocksdb.ttl.ms";
+  static final public String ROCKSDB_COMPACTION_STYLE = "rocksdb.compaction.style";
+  static final public String ROCKSDB_NUM_WRITE_BUFFERS = "rocksdb.num.write.buffers";
+  static final public String ROCKSDB_MAX_LOG_FILE_SIZE_BYTES = "rocksdb.max.log.file.size.bytes";
+  static final public String ROCKSDB_KEEP_LOG_FILE_NUM = "rocksdb.keep.log.file.num";
+
+  protected Integer writeBatchSize;
+  protected Integer objectCacheSize;
+  private Integer cacheSize;
+  private Integer writeBufferSize;
+  private Integer blockSize;
+  private Integer ttl;
+  private Integer numWriteBuffers;
+  private Integer maxLogFileSize;
+  private Integer numLogFilesToKeep;
+  private String compressionType;
+  private String compactionStyle;
+
+  public RocksDbTableDescriptor(String tableId) {
+    super(tableId);
+  }
+
+  /**
+   * Refer to <code>stores.store-name.write.batch.size</code> in Samza configuration guide
+   * @param writeBatchSize write batch size
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor withWriteBatchSize(int writeBatchSize) {
+    this.writeBatchSize = writeBatchSize;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.object.cache.size</code> in Samza configuration guide
+   * @param objectCacheSize the object cache size
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor withObjectCacheSize(int objectCacheSize) {
+    this.objectCacheSize = objectCacheSize;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.container.cache.size.bytes</code> in Samza configuration guide
+   * @param cacheSize the cache size in bytes
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withCacheSize(int cacheSize) {
+    this.cacheSize = cacheSize;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.container.write.buffer.size.bytes</code> in Samza configuration guide
+   * @param writeBufferSize the write buffer size in bytes
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withWriteBufferSize(int writeBufferSize) {
+    this.writeBufferSize = writeBufferSize;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.compression</code> in Samza configuration guide
+   * @param compressionType the compression type
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withCompressionType(String compressionType) {
+    this.compressionType = compressionType;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.block.size.bytes</code> in Samza configuration guide
+   * @param blockSize the block size in bytes
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withBlockSize(int blockSize) {
+    this.blockSize = blockSize;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.ttl.ms</code> in Samza configuration guide
+   * @param ttl the time to live in milliseconds
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withTtl(int ttl) {
+    this.ttl = ttl;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.compaction.style</code> in Samza configuration guide
+   * @param compactionStyle the compaction style
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withCompactionStyle(String compactionStyle) {
+    this.compactionStyle = compactionStyle;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.num.write.buffers</code> in Samza configuration guide
+   * @param numWriteBuffers the number of write buffers
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withNumWriteBuffers(int numWriteBuffers) {
+    this.numWriteBuffers = numWriteBuffers;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.max.log.file.size.bytes</code> in Samza configuration guide
+   * @param maxLogFileSize the maximal log file size in bytes
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withMaxLogFileSize(int maxLogFileSize) {
+    this.maxLogFileSize = maxLogFileSize;
+    return this;
+  }
+
+  /**
+   * Refer to <code>stores.store-name.rocksdb.num.write.buffers</code> in Samza configuration guide
+   * @param numLogFilesToKeep the number of log files to keep
+   * @return this table descriptor instance
+   */
+  public RocksDbTableDescriptor<K, V> withNumLogFilesToKeep(int numLogFilesToKeep) {
+    this.numLogFilesToKeep = numLogFilesToKeep;
+    return this;
+  }
+
+  /**
+   * Create a table spec based on this table description
+   * @return the table spec
+   */
+  @Override
+  public TableSpec getTableSpec() {
+
+    validate();
+
+    Map<String, String> tableSpecConfig = new HashMap<>();
+    generateTableSpecConfig(tableSpecConfig);
+
+    return new TableSpec(tableId, serde, RocksDbTableProviderFactory.class.getName(), tableSpecConfig);
+  }
+
+  @Override
+  protected void generateTableSpecConfig(Map<String, String> tableSpecConfig) {
+
+    super.generateTableSpecConfig(tableSpecConfig);
+
+    if (writeBatchSize != null) {
+      addRocksDbConfig(tableSpecConfig, WRITE_BATCH_SIZE, writeBatchSize.toString());
+    }
+    if (objectCacheSize != null) {
+      addRocksDbConfig(tableSpecConfig, OBJECT_CACHE_SIZE, objectCacheSize.toString());
+    }
+    if (cacheSize != null) {
+      addRocksDbConfig(tableSpecConfig, CONTAINER_CACHE_SIZE_BYTES, cacheSize.toString());
+    }
+    if (writeBufferSize != null) {
+      addRocksDbConfig(tableSpecConfig, CONTAINER_WRITE_BUFFER_SIZE_BYTES, writeBufferSize.toString());
+    }
+    if (compressionType != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_COMPRESSION, compressionType);
+    }
+    if (blockSize != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_BLOCK_SIZE_BYTES, blockSize.toString());
+    }
+    if (ttl != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_TTL_MS, ttl.toString());
+    }
+    if (compactionStyle != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_COMPACTION_STYLE, compactionStyle);
+    }
+    if (numWriteBuffers != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_NUM_WRITE_BUFFERS, numWriteBuffers.toString());
+    }
+    if (maxLogFileSize != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_MAX_LOG_FILE_SIZE_BYTES, maxLogFileSize.toString());
+    }
+    if (numLogFilesToKeep != null) {
+      addRocksDbConfig(tableSpecConfig, ROCKSDB_KEEP_LOG_FILE_NUM, numLogFilesToKeep.toString());
+    }
+  }
+
+  private void addRocksDbConfig(Map<String, String> map, String key, String value) {
+    map.put("rocksdb." + key, value);
+  }
+
+}
diff --git a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProvider.java b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProvider.java
new file mode 100644
index 0000000000..eb8188f06d
--- /dev/null
+++ b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProvider.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.StorageConfig;
+import org.apache.samza.table.TableSpec;
+
+
+/**
+ * Table provider for tables backed by RocksDb.
+ */
+public class RocksDbTableProvider extends BaseLocalStoreBackedTableProvider {
+
+  public RocksDbTableProvider(TableSpec tableSpec) {
+    super(tableSpec);
+  }
+
+  @Override
+  public Map<String, String> generateConfig(Map<String, String> config) {
+
+    Map<String, String> tableConfig = new HashMap<>();
+
+    // Store factory configuration
+    tableConfig.put(String.format(
+        StorageConfig.FACTORY(), tableSpec.getId()),
+        RocksDbKeyValueStorageEngineFactory.class.getName());
+
+    // Common store configuration
+    tableConfig.putAll(generateCommonStoreConfig(config));
+
+    // Rest of the configuration
+    tableSpec.getConfig().forEach((k, v) -> {
+      String realKey = k.startsWith("rocksdb.") ?
+          String.format("stores.%s", tableSpec.getId()) + "." + k.substring("rocksdb.".length())
+        : String.format(JavaTableConfig.TABLE_ID_PREFIX, tableSpec.getId()) + "." + k;
+      tableConfig.put(realKey, v);
+    });
+
+    logger.info("Generated configuration for table " + tableSpec.getId());
+
+    return tableConfig;
+  }
+
+}
diff --git a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProviderFactory.java b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProviderFactory.java
new file mode 100644
index 0000000000..dbe0f97a3c
--- /dev/null
+++ b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbTableProviderFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import org.apache.samza.table.TableProvider;
+import org.apache.samza.table.TableProviderFactory;
+import org.apache.samza.table.TableSpec;
+
+
+public class RocksDbTableProviderFactory implements TableProviderFactory {
+  @Override
+  public TableProvider getTableProvider(TableSpec tableSpec) {
+    return new RocksDbTableProvider(tableSpec);
+  }
+}
diff --git a/samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableDescriptor.java b/samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableDescriptor.java
new file mode 100644
index 0000000000..49fe6eb975
--- /dev/null
+++ b/samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableDescriptor.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import org.apache.samza.serializers.IntegerSerde;
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.serializers.StringSerde;
+import org.apache.samza.table.TableSpec;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+
+public class TestRocksDbTableDescriptor {
+
+  @Test
+  public void testMinimal() {
+    new RocksDbTableDescriptor<Integer, String>("1")
+        .validate();
+  }
+
+  @Test
+  public void testSerde() {
+    TableSpec tableSpec = new RocksDbTableDescriptor<Integer, String>("1")
+        .withSerde(KVSerde.of(new IntegerSerde(), new StringSerde()))
+        .getTableSpec();
+    Assert.assertNotNull(tableSpec.getSerde());
+    Assert.assertEquals(tableSpec.getSerde().getKeySerde().getClass(), IntegerSerde.class);
+    Assert.assertEquals(tableSpec.getSerde().getValueSerde().getClass(), StringSerde.class);
+  }
+
+  @Test
+  public void testTableSpec() {
+
+    TableSpec tableSpec = new RocksDbTableDescriptor<Integer, String>("1")
+        .withSerde(KVSerde.of(new IntegerSerde(), new StringSerde()))
+        .withBlockSize(1)
+        .withCacheSize(2)
+        .withCompactionStyle("fifo")
+        .withCompressionType("snappy")
+        .withMaxLogFileSize(3)
+        .withNumLogFilesToKeep(4)
+        .withNumWriteBuffers(5)
+        .withObjectCacheSize(6)
+        .withTtl(7)
+        .withWriteBatchSize(8)
+        .withWriteBufferSize(9)
+        .withConfig("rocksdb.abc", "xyz")
+        .getTableSpec();
+
+    Assert.assertNotNull(tableSpec.getSerde());
+    Assert.assertNotNull(tableSpec.getSerde().getKeySerde());
+    Assert.assertNotNull(tableSpec.getSerde().getValueSerde());
+    Assert.assertEquals("1", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_BLOCK_SIZE_BYTES));
+    Assert.assertEquals("2", getConfig(tableSpec, RocksDbTableDescriptor.CONTAINER_CACHE_SIZE_BYTES));
+    Assert.assertEquals("3", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_MAX_LOG_FILE_SIZE_BYTES));
+    Assert.assertEquals("4", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_KEEP_LOG_FILE_NUM));
+    Assert.assertEquals("5", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_NUM_WRITE_BUFFERS));
+    Assert.assertEquals("6", getConfig(tableSpec, RocksDbTableDescriptor.OBJECT_CACHE_SIZE));
+    Assert.assertEquals("7", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_TTL_MS));
+    Assert.assertEquals("8", getConfig(tableSpec, RocksDbTableDescriptor.WRITE_BATCH_SIZE));
+    Assert.assertEquals("9", getConfig(tableSpec, RocksDbTableDescriptor.CONTAINER_WRITE_BUFFER_SIZE_BYTES));
+    Assert.assertEquals("snappy", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_COMPRESSION));
+    Assert.assertEquals("fifo", getConfig(tableSpec, RocksDbTableDescriptor.ROCKSDB_COMPACTION_STYLE));
+    Assert.assertEquals("xyz", getConfig(tableSpec, "abc"));
+  }
+
+  private String getConfig(TableSpec tableSpec, String key) {
+    return tableSpec.getConfig().get("rocksdb." + key);
+  }
+}
diff --git a/samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableProvider.java b/samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableProvider.java
new file mode 100644
index 0000000000..beda5da49c
--- /dev/null
+++ b/samza-kv-rocksdb/src/test/java/org/apache/samza/storage/kv/TestRocksDbTableProvider.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.storage.kv;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.StorageConfig;
+import org.apache.samza.serializers.IntegerSerde;
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.table.TableProvider;
+import org.apache.samza.table.TableSpec;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+
+public class TestRocksDbTableProvider {
+  @Test
+  public void testGenerateConfig() {
+
+    Map<String, String> tableSpecConfig = new HashMap<>();
+    tableSpecConfig.put("rocksdb.c1", "c1-value");
+    tableSpecConfig.put("rocksdb.c2", "c2-value");
+    tableSpecConfig.put("c3", "c3-value");
+    tableSpecConfig.put("c4", "c4-value");
+
+    TableSpec tableSpec = new TableSpec("t1", KVSerde.of(new IntegerSerde(), new IntegerSerde()),
+        "my-table-provider-factory", tableSpecConfig);
+
+    Map<String, String> config = new HashMap<>();
+    config.put(String.format(JavaTableConfig.TABLE_KEY_SERDE, "t1"), "ks1");
+    config.put(String.format(JavaTableConfig.TABLE_VALUE_SERDE, "t1"), "vs1");
+
+    TableProvider tableProvider = new RocksDbTableProvider(tableSpec);
+    Map<String, String> tableConfig = tableProvider.generateConfig(config);
+
+    Assert.assertEquals("ks1", tableConfig.get(String.format(StorageConfig.KEY_SERDE(), "t1")));
+    Assert.assertEquals("vs1", tableConfig.get(String.format(StorageConfig.MSG_SERDE(), "t1")));
+    Assert.assertEquals(
+        RocksDbKeyValueStorageEngineFactory.class.getName(),
+        tableConfig.get(String.format(StorageConfig.FACTORY(), "t1")));
+    Assert.assertEquals("c1-value", tableConfig.get("stores.t1.c1"));
+    Assert.assertEquals("c2-value", tableConfig.get("stores.t1.c2"));
+    Assert.assertEquals("c3-value", tableConfig.get("tables.t1.c3"));
+    Assert.assertEquals("c4-value", tableConfig.get("tables.t1.c4"));
+  }
+}
diff --git a/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableDescriptor.java b/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableDescriptor.java
new file mode 100644
index 0000000000..1f9b57b854
--- /dev/null
+++ b/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableDescriptor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import java.util.Map;
+
+import org.apache.samza.operators.BaseTableDescriptor;
+
+
+/**
+ * Table descriptor for store backed tables.
+ *
+ * @param <K> the type of the key in this table
+ * @param <V> the type of the value in this table
+ * @param <D> the type of the concrete table descriptor
+ */
+abstract public class BaseLocalStoreBackedTableDescriptor<K, V, D extends BaseLocalStoreBackedTableDescriptor<K, V, D>>
+    extends BaseTableDescriptor<K, V, D> {
+
+  /**
+   * Constructs a table descriptor instance
+   * @param tableId Id of the table
+   */
+  public BaseLocalStoreBackedTableDescriptor(String tableId) {
+    super(tableId);
+  }
+
+  @Override
+  protected void generateTableSpecConfig(Map<String, String> tableSpecConfig) {
+    super.generateTableSpecConfig(tableSpecConfig);
+  }
+
+  /**
+   * Validate that this table descriptor is constructed properly
+   */
+  protected void validate() {
+    super.validate();
+  }
+
+}
diff --git a/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableProvider.java b/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableProvider.java
new file mode 100644
index 0000000000..4af0f1dc78
--- /dev/null
+++ b/samza-kv/src/main/java/org/apache/samza/storage/kv/BaseLocalStoreBackedTableProvider.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.MapConfig;
+import org.apache.samza.config.StorageConfig;
+import org.apache.samza.storage.StorageEngine;
+import org.apache.samza.table.LocalStoreBackedTableProvider;
+import org.apache.samza.table.Table;
+import org.apache.samza.table.TableSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Base class for tables backed by Samza stores, see {@link LocalStoreBackedTableProvider}.
+ */
+abstract public class BaseLocalStoreBackedTableProvider implements LocalStoreBackedTableProvider {
+
+  protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+  protected final TableSpec tableSpec;
+
+  protected KeyValueStore kvStore;
+
+  public BaseLocalStoreBackedTableProvider(TableSpec tableSpec) {
+    this.tableSpec = tableSpec;
+  }
+
+  @Override
+  public void init(StorageEngine store) {
+    kvStore = (KeyValueStore) store;
+    logger.info("Initialized backing store for table " + tableSpec.getId());
+  }
+
+  @Override
+  public Table getTable() {
+    if (kvStore == null) {
+      throw new SamzaException("Store not initialized for table " + tableSpec.getId());
+    }
+    return new LocalStoreBackedReadWriteTable(kvStore);
+  }
+
+  @Override
+  public void start() {
+    logger.info("Starting table provider for table " + tableSpec.getId());
+  }
+
+  @Override
+  public void stop() {
+    logger.info("Stopping table provider for table " + tableSpec.getId());
+  }
+
+  protected Map<String, String> generateCommonStoreConfig(Map<String, String> config) {
+
+    Map<String, String> storeConfig = new HashMap<>();
+
+    // We assume the configuration for serde are already generated for this table,
+    // so we simply carry them over to store configuration.
+    //
+    JavaTableConfig tableConfig = new JavaTableConfig(new MapConfig(config));
+
+    String keySerde = tableConfig.getKeySerde(tableSpec.getId());
+    storeConfig.put(String.format(StorageConfig.KEY_SERDE(), tableSpec.getId()), keySerde);
+
+    String valueSerde = tableConfig.getValueSerde(tableSpec.getId());
+    storeConfig.put(String.format(StorageConfig.MSG_SERDE(), tableSpec.getId()), valueSerde);
+
+    return storeConfig;
+  }
+}
diff --git a/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadWriteTable.java b/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadWriteTable.java
new file mode 100644
index 0000000000..3149c86eaf
--- /dev/null
+++ b/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadWriteTable.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import java.util.List;
+
+import org.apache.samza.table.ReadWriteTable;
+
+
+/**
+ * A store backed readable and writable table
+ *
+ * @param <K> the type of the key in this table
+ * @param <V> the type of the value in this table
+ */
+public class LocalStoreBackedReadWriteTable<K, V> extends LocalStoreBackedReadableTable<K, V>
+    implements ReadWriteTable<K, V> {
+
+  /**
+   * Constructs an instance of {@link LocalStoreBackedReadWriteTable}
+   * @param kvStore the backing store
+   */
+  public LocalStoreBackedReadWriteTable(KeyValueStore kvStore) {
+    super(kvStore);
+  }
+
+  @Override
+  public void put(K key, V value) {
+    kvStore.put(key, value);
+  }
+
+  @Override
+  public void putAll(List<Entry<K, V>> entries) {
+    entries.forEach(e -> kvStore.put(e.getKey(), e.getValue()));
+  }
+
+  @Override
+  public void delete(K key) {
+    kvStore.delete(key);
+  }
+
+  @Override
+  public void deleteAll(List<K> keys) {
+    keys.forEach(k -> kvStore.delete(k));
+  }
+
+  @Override
+  public void flush() {
+    kvStore.flush();
+  }
+
+}
diff --git a/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadableTable.java b/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadableTable.java
new file mode 100644
index 0000000000..fead086348
--- /dev/null
+++ b/samza-kv/src/main/java/org/apache/samza/storage/kv/LocalStoreBackedReadableTable.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.storage.kv;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.samza.table.ReadableTable;
+
+
+/**
+ * A store backed readable table
+ *
+ * @param <K> the type of the key in this table
+ * @param <V> the type of the value in this table
+ */
+public class LocalStoreBackedReadableTable<K, V> implements ReadableTable<K, V> {
+
+  protected KeyValueStore<K, V> kvStore;
+
+  /**
+   * Constructs an instance of {@link LocalStoreBackedReadableTable}
+   * @param kvStore the backing store
+   */
+  public LocalStoreBackedReadableTable(KeyValueStore<K, V> kvStore) {
+    this.kvStore = kvStore;
+  }
+
+  @Override
+  public V get(K key) {
+    return kvStore.get(key);
+  }
+
+  @Override
+  public Map<K, V> getAll(List<K> keys) {
+    return keys.stream().collect(Collectors.toMap(k -> k, k -> kvStore.get(k)));
+  }
+
+  @Override
+  public void close() {
+    // The KV store is not closed here as it may still be needed by downstream operators,
+    // it will be closed by the SamzaContainer
+  }
+}
diff --git a/samza-kv/src/test/java/org/apache/samza/storage/kv/TestLocalBaseStoreBackedTableProvider.java b/samza-kv/src/test/java/org/apache/samza/storage/kv/TestLocalBaseStoreBackedTableProvider.java
new file mode 100644
index 0000000000..9c95637834
--- /dev/null
+++ b/samza-kv/src/test/java/org/apache/samza/storage/kv/TestLocalBaseStoreBackedTableProvider.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.storage.kv;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.JavaTableConfig;
+import org.apache.samza.config.StorageConfig;
+import org.apache.samza.storage.StorageEngine;
+import org.apache.samza.table.TableSpec;
+import org.junit.Before;
+import org.junit.Test;
+
+import junit.framework.Assert;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class TestLocalBaseStoreBackedTableProvider {
+
+  private BaseLocalStoreBackedTableProvider tableProvider;
+
+  @Before
+  public void prepare() {
+    TableSpec tableSpec = mock(TableSpec.class);
+    when(tableSpec.getId()).thenReturn("t1");
+    tableProvider = new BaseLocalStoreBackedTableProvider(tableSpec) {
+      @Override
+      public Map<String, String> generateConfig(Map<String, String> config) {
+        return generateCommonStoreConfig(config);
+      }
+    };
+  }
+
+  @Test
+  public void testInit() {
+    StorageEngine store = mock(KeyValueStorageEngine.class);
+    tableProvider.init(store);
+    Assert.assertNotNull(tableProvider.getTable());
+  }
+
+  @Test(expected = SamzaException.class)
+  public void testInitFail() {
+    Assert.assertNotNull(tableProvider.getTable());
+  }
+
+  @Test
+  public void testGenerateCommonStoreConfig() {
+    Map<String, String> config = new HashMap<>();
+    config.put(String.format(JavaTableConfig.TABLE_KEY_SERDE, "t1"), "ks1");
+    config.put(String.format(JavaTableConfig.TABLE_VALUE_SERDE, "t1"), "vs1");
+
+    Map<String, String> tableConfig = tableProvider.generateConfig(config);
+    Assert.assertEquals("ks1", tableConfig.get(String.format(StorageConfig.KEY_SERDE(), "t1")));
+    Assert.assertEquals("vs1", tableConfig.get(String.format(StorageConfig.MSG_SERDE(), "t1")));
+  }
+}
diff --git a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTable.java b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTable.java
new file mode 100644
index 0000000000..8f7eb5d3fd
--- /dev/null
+++ b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTable.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.test.table;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.samza.application.StreamApplication;
+import org.apache.samza.config.Config;
+import org.apache.samza.config.JobConfig;
+import org.apache.samza.config.JobCoordinatorConfig;
+import org.apache.samza.config.MapConfig;
+import org.apache.samza.config.TaskConfig;
+import org.apache.samza.container.grouper.task.SingleContainerGrouperFactory;
+import org.apache.samza.operators.KV;
+import org.apache.samza.operators.MessageStream;
+import org.apache.samza.operators.functions.MapFunction;
+import org.apache.samza.operators.functions.StreamTableJoinFunction;
+import org.apache.samza.runtime.LocalApplicationRunner;
+import org.apache.samza.serializers.IntegerSerde;
+import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.serializers.NoOpSerde;
+import org.apache.samza.standalone.PassthroughCoordinationUtilsFactory;
+import org.apache.samza.standalone.PassthroughJobCoordinatorFactory;
+import org.apache.samza.storage.kv.inmemory.InMemoryTableDescriptor;
+import org.apache.samza.table.ReadableTable;
+import org.apache.samza.table.Table;
+import org.apache.samza.task.TaskContext;
+import org.apache.samza.test.harness.AbstractIntegrationTestHarness;
+import org.apache.samza.test.table.TestTableData.EnrichedPageView;
+import org.apache.samza.test.table.TestTableData.PageView;
+import org.apache.samza.test.table.TestTableData.PageViewJsonSerde;
+import org.apache.samza.test.table.TestTableData.PageViewJsonSerdeFactory;
+import org.apache.samza.test.table.TestTableData.Profile;
+import org.apache.samza.test.table.TestTableData.ProfileJsonSerde;
+import org.apache.samza.test.util.ArraySystemFactory;
+import org.apache.samza.test.util.Base64Serializer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test class tests sendTo() and join() for local tables
+ */
+public class TestLocalTable extends AbstractIntegrationTestHarness {
+
+  @Test
+  public void testSendTo() throws  Exception {
+
+    int count = 10;
+    Profile[] profiles = TestTableData.generateProfiles(count);
+
+    int partitionCount = 4;
+    Map<String, String> configs = getBaseJobConfig();
+
+    configs.put("streams.Profile.samza.system", "test");
+    configs.put("streams.Profile.source", Base64Serializer.serialize(profiles));
+    configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount));
+
+    MyMapFunction mapFn = new MyMapFunction();
+
+    final LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(configs));
+    final StreamApplication app = (streamGraph, cfg) -> {
+
+      Table<KV<Integer, Profile>> table = streamGraph.getTable(new InMemoryTableDescriptor("t1")
+          .withSerde(KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
+
+      streamGraph.getInputStream("Profile", new NoOpSerde<Profile>())
+          .map(mapFn)
+          .sendTo(table);
+    };
+
+    runner.run(app);
+    runner.waitForFinish();
+
+    assertEquals(count * partitionCount, mapFn.received.size());
+    assertEquals(count, new HashSet(mapFn.received).size());
+    mapFn.received.forEach(p -> Assert.assertTrue(mapFn.table.get(p.getMemberId()) != null));
+  }
+
+  @Test
+  public void testStreamTableJoin() throws Exception {
+
+    List<PageView> received = new LinkedList<>();
+    List<EnrichedPageView> joined = new LinkedList<>();
+
+    int count = 10;
+    PageView[] pageViews = TestTableData.generatePageViews(count);
+    Profile[] profiles = TestTableData.generateProfiles(count);
+
+    int partitionCount = 4;
+    Map<String, String> configs = getBaseJobConfig();
+
+    configs.put("streams.PageView.samza.system", "test");
+    configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews));
+    configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount));
+
+    configs.put("streams.Profile.samza.system", "test");
+    configs.put("streams.Profile.samza.bootstrap", "true");
+    configs.put("streams.Profile.source", Base64Serializer.serialize(profiles));
+    configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount));
+
+    final LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(configs));
+    final StreamApplication app = (streamGraph, cfg) -> {
+
+      Table<KV<Integer, Profile>> table = streamGraph.getTable(new InMemoryTableDescriptor("t1")
+          .withSerde(KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
+
+      streamGraph.getInputStream("Profile", new NoOpSerde<Profile>())
+          .map(m -> new KV(m.getMemberId(), m))
+          .sendTo(table);
+
+      streamGraph.getInputStream("PageView", new NoOpSerde<PageView>())
+          .map(pv -> {
+              received.add(pv);
+              return pv;
+            })
+          .partitionBy(PageView::getMemberId, v -> v, "p1")
+          .join(table, new PageViewToProfileJoinFunction())
+          .sink((m, collector, coordinator) -> joined.add(m));
+    };
+
+    runner.run(app);
+    runner.waitForFinish();
+
+    assertEquals(count * partitionCount, received.size());
+    assertEquals(count * partitionCount, joined.size());
+    assertTrue(joined.get(0) instanceof EnrichedPageView);
+  }
+
+  @Test
+  public void testDualStreamTableJoin() throws Exception {
+
+    List<Profile> sentToProfileTable1 = new LinkedList<>();
+    List<Profile> sentToProfileTable2 = new LinkedList<>();
+    List<EnrichedPageView> joinedPageViews1 = new LinkedList<>();
+    List<EnrichedPageView> joinedPageViews2 = new LinkedList<>();
+
+    KVSerde<Integer, Profile> profileKVSerde = KVSerde.of(new IntegerSerde(), new ProfileJsonSerde());
+    KVSerde<Integer, PageView> pageViewKVSerde = KVSerde.of(new IntegerSerde(), new PageViewJsonSerde());
+
+    PageViewToProfileJoinFunction joinFn1 = new PageViewToProfileJoinFunction();
+    PageViewToProfileJoinFunction joinFn2 = new PageViewToProfileJoinFunction();
+
+    int count = 10;
+    PageView[] pageViews = TestTableData.generatePageViews(count);
+    Profile[] profiles = TestTableData.generateProfiles(count);
+
+    int partitionCount = 4;
+    Map<String, String> configs = getBaseJobConfig();
+
+    configs.put("streams.Profile1.samza.system", "test");
+    configs.put("streams.Profile1.source", Base64Serializer.serialize(profiles));
+    configs.put("streams.Profile1.samza.bootstrap", "true");
+    configs.put("streams.Profile1.partitionCount", String.valueOf(partitionCount));
+
+    configs.put("streams.Profile2.samza.system", "test");
+    configs.put("streams.Profile2.source", Base64Serializer.serialize(profiles));
+    configs.put("streams.Profile2.samza.bootstrap", "true");
+    configs.put("streams.Profile2.partitionCount", String.valueOf(partitionCount));
+
+    configs.put("streams.PageView1.samza.system", "test");
+    configs.put("streams.PageView1.source", Base64Serializer.serialize(pageViews));
+    configs.put("streams.PageView1.partitionCount", String.valueOf(partitionCount));
+
+    configs.put("streams.PageView2.samza.system", "test");
+    configs.put("streams.PageView2.source", Base64Serializer.serialize(pageViews));
+    configs.put("streams.PageView2.partitionCount", String.valueOf(partitionCount));
+
+    final LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(configs));
+    final StreamApplication app = (streamGraph, cfg) -> {
+
+      Table<KV<Integer, Profile>> profileTable = streamGraph.getTable(new InMemoryTableDescriptor("t1")
+          .withSerde(KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
+
+      MessageStream<Profile> profileStream1 = streamGraph.getInputStream("Profile1", new NoOpSerde<Profile>());
+      MessageStream<Profile> profileStream2 = streamGraph.getInputStream("Profile2", new NoOpSerde<Profile>());
+
+      profileStream1
+          .map(m -> {
+              sentToProfileTable1.add(m);
+              return new KV(m.getMemberId(), m);
+            })
+          .sendTo(profileTable);
+      profileStream2
+          .map(m -> {
+              sentToProfileTable2.add(m);
+              return new KV(m.getMemberId(), m);
+            })
+          .sendTo(profileTable);
+
+      MessageStream<PageView> pageViewStream1 = streamGraph.getInputStream("PageView1", new NoOpSerde<PageView>());
+      MessageStream<PageView> pageViewStream2 = streamGraph.getInputStream("PageView2", new NoOpSerde<PageView>());
+
+      pageViewStream1
+          .partitionBy(PageView::getMemberId, v -> v, pageViewKVSerde, "p1")
+          .join(profileTable, joinFn1)
+          .sink((m, collector, coordinator) -> joinedPageViews1.add(m));
+
+      pageViewStream2
+          .partitionBy(PageView::getMemberId, v -> v, pageViewKVSerde, "p2")
+          .join(profileTable, joinFn2)
+          .sink((m, collector, coordinator) -> joinedPageViews2.add(m));
+    };
+
+    runner.run(app);
+    runner.waitForFinish();
+
+    assertEquals(count * partitionCount, sentToProfileTable1.size());
+    assertEquals(count * partitionCount, sentToProfileTable2.size());
+    assertEquals(count * partitionCount, joinFn1.count);
+    assertEquals(count * partitionCount, joinFn2.count);
+    assertEquals(count * partitionCount, joinedPageViews1.size());
+    assertEquals(count * partitionCount, joinedPageViews2.size());
+    assertTrue(joinedPageViews1.get(0) instanceof EnrichedPageView);
+    assertTrue(joinedPageViews2.get(0) instanceof EnrichedPageView);
+  }
+
+  private Map<String, String> getBaseJobConfig() {
+    Map<String, String> configs = new HashMap<>();
+    configs.put("systems.test.samza.factory", ArraySystemFactory.class.getName());
+
+    configs.put(JobConfig.JOB_NAME(), "test-table-job");
+    configs.put(JobConfig.PROCESSOR_ID(), "1");
+    configs.put(JobCoordinatorConfig.JOB_COORDINATION_UTILS_FACTORY, PassthroughCoordinationUtilsFactory.class.getName());
+    configs.put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, PassthroughJobCoordinatorFactory.class.getName());
+    configs.put(TaskConfig.GROUPER_FACTORY(), SingleContainerGrouperFactory.class.getName());
+
+    // For intermediate streams
+    configs.put("systems.kafka.samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory");
+    configs.put("systems.kafka.producer.bootstrap.servers", bootstrapUrl());
+    configs.put("systems.kafka.consumer.zookeeper.connect", zkConnect());
+    configs.put("systems.kafka.samza.key.serde", "int");
+    configs.put("systems.kafka.samza.msg.serde", "json");
+    configs.put("systems.kafka.default.stream.replication.factor", "1");
+    configs.put("job.default.system", "kafka");
+
+    configs.put("serializers.registry.int.class", "org.apache.samza.serializers.IntegerSerdeFactory");
+    configs.put("serializers.registry.json.class", PageViewJsonSerdeFactory.class.getName());
+
+    return configs;
+  }
+
+  private class MyMapFunction implements MapFunction<Profile, KV<Integer, Profile>> {
+
+    private List<Profile> received = new ArrayList<>();
+    private ReadableTable table;
+
+    @Override
+    public void init(Config config, TaskContext context) {
+      table = (ReadableTable) context.getTable("t1");
+    }
+
+    @Override
+    public KV<Integer, Profile> apply(Profile profile) {
+      received.add(profile);
+      return new KV(profile.getMemberId(), profile);
+    }
+  }
+
+  private class PageViewToProfileJoinFunction implements StreamTableJoinFunction
+      <Integer, KV<Integer, PageView>, KV<Integer, Profile>, EnrichedPageView> {
+    private int count;
+    @Override
+    public EnrichedPageView apply(KV<Integer, PageView> m, KV<Integer, Profile> r) {
+      ++count;
+      return r == null ? null :
+          new EnrichedPageView(m.getValue().getPageKey(), m.getKey(), r.getValue().getCompany());
+    }
+
+    @Override
+    public Integer getMessageKey(KV<Integer, PageView> message) {
+      return message.getKey();
+    }
+
+    @Override
+    public Integer getRecordKey(KV<Integer, Profile> record) {
+      return record.getKey();
+    }
+  }
+}
diff --git a/samza-test/src/test/java/org/apache/samza/test/table/TestTableData.java b/samza-test/src/test/java/org/apache/samza/test/table/TestTableData.java
new file mode 100644
index 0000000000..dfd0d1ba77
--- /dev/null
+++ b/samza-test/src/test/java/org/apache/samza/test/table/TestTableData.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.test.table;
+
+import java.io.Serializable;
+import java.util.Random;
+
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
+import org.apache.samza.serializers.Serde;
+import org.apache.samza.serializers.SerdeFactory;
+import org.codehaus.jackson.annotate.JsonCreator;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+
+
+public class TestTableData {
+
+  public static class PageView implements Serializable {
+    @JsonProperty("pageKey")
+    final String pageKey;
+    @JsonProperty("memberId")
+    final int memberId;
+
+    @JsonProperty("pageKey")
+    public String getPageKey() {
+      return pageKey;
+    }
+
+    @JsonProperty("memberId")
+    public int getMemberId() {
+      return memberId;
+    }
+
+    @JsonCreator
+    public PageView(@JsonProperty("pageKey") String pageKey, @JsonProperty("memberId") int memberId) {
+      this.pageKey = pageKey;
+      this.memberId = memberId;
+    }
+  }
+
+  public static class Profile implements Serializable {
+    @JsonProperty("memberId")
+    final int memberId;
+
+    @JsonProperty("company")
+    final String company;
+
+    @JsonProperty("memberId")
+    public int getMemberId() {
+      return memberId;
+    }
+
+    @JsonProperty("company")
+    public String getCompany() {
+      return company;
+    }
+
+    @JsonCreator
+    public Profile(@JsonProperty("memberId") int memberId, @JsonProperty("company") String company) {
+      this.memberId = memberId;
+      this.company = company;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o == null || !(o instanceof Profile)) {
+        return false;
+      }
+      return ((Profile) o).getMemberId() == memberId;
+    }
+
+    @Override
+    public int hashCode() {
+      return memberId;
+    }
+  }
+
+  public static class EnrichedPageView extends PageView {
+
+    @JsonProperty("company")
+    final String company;
+
+    @JsonProperty("company")
+    public String getCompany() {
+      return company;
+    }
+
+    @JsonCreator
+    public EnrichedPageView(
+        @JsonProperty("pageKey") String pageKey,
+        @JsonProperty("memberId") int memberId,
+        @JsonProperty("company") String company) {
+      super(pageKey, memberId);
+      this.company = company;
+    }
+  }
+
+  public static class PageViewJsonSerdeFactory implements SerdeFactory<PageView> {
+    @Override public Serde<PageView> getSerde(String name, Config config) {
+      return new PageViewJsonSerde();
+    }
+  }
+
+  public static class ProfileJsonSerdeFactory implements SerdeFactory<Profile> {
+    @Override public Serde<Profile> getSerde(String name, Config config) {
+      return new ProfileJsonSerde();
+    }
+  }
+
+  public static class PageViewJsonSerde implements Serde<PageView> {
+
+    @Override
+    public PageView fromBytes(byte[] bytes) {
+      try {
+        ObjectMapper mapper = new ObjectMapper();
+        return mapper.readValue(new String(bytes, "UTF-8"), new TypeReference<PageView>() { });
+      } catch (Exception e) {
+        throw new SamzaException(e);
+      }
+    }
+
+    @Override
+    public byte[] toBytes(PageView pv) {
+      try {
+        ObjectMapper mapper = new ObjectMapper();
+        return mapper.writeValueAsString(pv).getBytes("UTF-8");
+      } catch (Exception e) {
+        throw new SamzaException(e);
+      }
+    }
+  }
+
+  public static class ProfileJsonSerde implements Serde<Profile> {
+
+    @Override
+    public Profile fromBytes(byte[] bytes) {
+      try {
+        ObjectMapper mapper = new ObjectMapper();
+        return mapper.readValue(new String(bytes, "UTF-8"), new TypeReference<Profile>() { });
+      } catch (Exception e) {
+        throw new SamzaException(e);
+      }
+    }
+
+    @Override
+    public byte[] toBytes(Profile p) {
+      try {
+        ObjectMapper mapper = new ObjectMapper();
+        return mapper.writeValueAsString(p).getBytes("UTF-8");
+      } catch (Exception e) {
+        throw new SamzaException(e);
+      }
+    }
+  }
+
+  private static final String[] PAGEKEYS = {"inbox", "home", "search", "pymk", "group", "job"};
+
+  static public PageView[] generatePageViews(int count) {
+    Random random = new Random();
+    PageView[] pageviews = new PageView[count];
+    for (int i = 0; i < count; i++) {
+      String pagekey = PAGEKEYS[random.nextInt(PAGEKEYS.length - 1)];
+      int memberId = random.nextInt(10);
+      pageviews[i] = new PageView(pagekey, memberId);
+    }
+    return pageviews;
+  }
+
+  private static final String[] COMPANIES = {"MSFT", "LKND", "GOOG", "FB", "AMZN", "CSCO"};
+
+  static public Profile[] generateProfiles(int count) {
+    Random random = new Random();
+    Profile[] profiles = new Profile[count];
+    for (int i = 0; i < count; i++) {
+      String company = COMPANIES[random.nextInt(COMPANIES.length - 1)];
+      profiles[i] = new Profile(i, company);
+    }
+    return profiles;
+  }
+
+}
diff --git a/samza-test/src/test/java/org/apache/samza/test/util/ArraySystemConsumer.java b/samza-test/src/test/java/org/apache/samza/test/util/ArraySystemConsumer.java
index 832457b121..6ba28ae50c 100644
--- a/samza-test/src/test/java/org/apache/samza/test/util/ArraySystemConsumer.java
+++ b/samza-test/src/test/java/org/apache/samza/test/util/ArraySystemConsumer.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import org.apache.samza.config.Config;
 import org.apache.samza.system.IncomingMessageEnvelope;
@@ -58,9 +59,10 @@ public void register(SystemStreamPartition systemStreamPartition, String s) {
   public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> set, long l) throws InterruptedException {
     if (!done) {
       Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopeMap = new HashMap<>();
+      final AtomicInteger offset = new AtomicInteger(0);
       set.forEach(ssp -> {
           List<IncomingMessageEnvelope> envelopes = Arrays.stream(getArrayObjects(ssp.getSystemStream().getStream(), config))
-              .map(object -> new IncomingMessageEnvelope(ssp, null, null, object)).collect(Collectors.toList());
+              .map(object -> new IncomingMessageEnvelope(ssp, String.valueOf(offset.incrementAndGet()), null, object)).collect(Collectors.toList());
           envelopes.add(IncomingMessageEnvelope.buildEndOfStreamEnvelope(ssp));
           envelopeMap.put(ssp, envelopes);
         });
diff --git a/samza-test/src/test/java/org/apache/samza/test/util/SimpleSystemAdmin.java b/samza-test/src/test/java/org/apache/samza/test/util/SimpleSystemAdmin.java
index 8890a2f066..c735c74f82 100644
--- a/samza-test/src/test/java/org/apache/samza/test/util/SimpleSystemAdmin.java
+++ b/samza-test/src/test/java/org/apache/samza/test/util/SimpleSystemAdmin.java
@@ -24,7 +24,9 @@
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+
 import org.apache.samza.Partition;
+import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
 import org.apache.samza.system.SystemAdmin;
 import org.apache.samza.system.SystemStreamMetadata;
@@ -50,13 +52,17 @@ public Map<SystemStreamPartition, String> getOffsetsAfter(Map<SystemStreamPartit
   @Override
   public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
     return streamNames.stream()
-        .collect(Collectors.toMap(
-            Function.<String>identity(),
-            streamName -> {
+        .collect(Collectors.toMap(Function.identity(), streamName -> {
+            int messageCount = isBootstrapStream(streamName) ? getMessageCount(streamName) : -1;
+            String oldestOffset = messageCount < 0 ? null : "0";
+            String newestOffset = messageCount < 0 ? null : String.valueOf(messageCount - 1);
+            String upcomingOffset = messageCount < 0 ? null : String.valueOf(messageCount);
             Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> metadataMap = new HashMap<>();
             int partitionCount = config.getInt("streams." + streamName + ".partitionCount", 1);
             for (int i = 0; i < partitionCount; i++) {
-              metadataMap.put(new Partition(i), new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null));
+              metadataMap.put(new Partition(i), new SystemStreamMetadata.SystemStreamPartitionMetadata(
+                  oldestOffset, newestOffset, upcomingOffset
+              ));
             }
             return new SystemStreamMetadata(streamName, metadataMap);
           }));
@@ -71,5 +77,17 @@ public Integer offsetComparator(String offset1, String offset2) {
     }
     return offset1.compareTo(offset2);
   }
+
+  private int getMessageCount(String streamName) {
+    try {
+      return Base64Serializer.deserialize(config.get("streams." + streamName + ".source"), Object[].class).length;
+    } catch (Exception e) {
+      throw new SamzaException(e);
+    }
+  }
+
+  private boolean isBootstrapStream(String streamName) {
+    return "true".equalsIgnoreCase(config.get("streams." + streamName + ".samza.bootstrap", "false"));
+  }
 }
 

From 1d420e750eb8dc7f0715f7e11e01af1e9ce61506 Mon Sep 17 00:00:00 2001
From: Daniel Chen <29577458+dxichen@users.noreply.github.com>
Date: Mon, 11 Dec 2017 16:45:44 -0800
Subject: [PATCH 08/36] SAMZA-1532; Eventhub connector fix

Key fixes vjagadish1989 lhaiesp srinipunuru
- Switched Producer source vs destination assumptions in `send`, `register`
- Check `OME.key` if `OME.partitionId` is null for to get partitionId
- Upcoming offset changed the `END_OF_STREAM` rather than `newestOffset` + 1, eventHub returns an error if the offset does not exist in the system
- Made the NewestOffset+1 as upcoming offset, consumer checks if the offset is valid on startup
- Differentiated between streamNames and streamIds in configs, consumer, producer
- Checkpoint table named after job name
- Checkpoint prints better message for invalid key on write

QOL
- How to ignore integration tests
- Improved logging

EDIT:
- Also added Round Robin producer partitioning

Author: Daniel Chen <29577458+dxichen@users.noreply.github.com>

Reviewers: Jagadish<jagadish@apache.org>

Closes #377 from dxichen/eventhub-connector-fix
---
 build.gradle                                  |   5 -
 .../azure/AzureCheckpointManager.java         |  31 +++++-
 .../azure/AzureCheckpointManagerFactory.java  |   4 +-
 .../samza/system/eventhub/EventHubConfig.java |  71 ++++++++++--
 .../eventhub/SamzaEventHubClientManager.java  |   4 +-
 .../eventhub/admin/EventHubSystemAdmin.java   |  17 +--
 .../consumer/EventHubSystemConsumer.java      |  77 +++++++++----
 .../producer/EventHubSystemProducer.java      | 103 ++++++++++--------
 .../azure/ITestAzureCheckpointManager.java    |   2 +-
 .../MockEventHubClientManagerFactory.java     |  18 +++
 .../admin/TestEventHubSystemAdmin.java        |  15 +--
 .../consumer/ITestEventHubSystemConsumer.java |   5 +-
 .../consumer/TestEventHubSystemConsumer.java  |  19 +++-
 .../producer/ITestEventHubSystemProducer.java |   2 +
 .../producer/TestEventHubSystemProducer.java  |  34 +++---
 15 files changed, 272 insertions(+), 135 deletions(-)

diff --git a/build.gradle b/build.gradle
index be1baf7b8d..330ff0b5e3 100644
--- a/build.gradle
+++ b/build.gradle
@@ -213,11 +213,6 @@ project(':samza-azure') {
     configFile = new File(rootDir, "checkstyle/checkstyle.xml")
     toolVersion = "$checkstyleVersion"
   }
-  test {
-    // Exclude integration tests that require connection to EventHub
-    exclude 'org/apache/samza/system/eventhub/producer/*ITest*'
-    exclude 'org/apache/samza/system/eventhub/consumer/*ITest*'
-  }
 }
 
 project(':samza-aws') {
diff --git a/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java b/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java
index df3e4904ce..2cad3bd6ee 100644
--- a/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java
+++ b/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManager.java
@@ -34,6 +34,7 @@
 import org.apache.samza.system.SystemStreamPartition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import scala.Option;
 
 import java.net.URISyntaxException;
 import java.util.HashMap;
@@ -60,12 +61,16 @@ public class AzureCheckpointManager implements CheckpointManager {
   private static final Logger LOG = LoggerFactory.getLogger(AzureCheckpointManager.class.getName());
   private static final String PARTITION_KEY = "PartitionKey";
 
+  // Invalid characters in key field on Azure Table
+  public static final String REGEX_INVALID_KEY = ".*[#?/\\\\].*";
+  public static final String REGEX_TABLE_NAME = "[^A-Za-z0-9]";
+
   public static final int MAX_WRITE_BATCH_SIZE = 100;
-  public static final String CHECKPOINT_MANAGER_TABLE_NAME = "SamzaTaskCheckpoints";
   public static final String SYSTEM_PROP_NAME = "system";
   public static final String STREAM_PROP_NAME = "stream";
   public static final String PARTITION_PROP_NAME = "partition";
 
+  private final String jobTableName;
   private final String storageConnectionString;
   private final AzureClient azureClient;
   private CloudTable cloudTable;
@@ -73,7 +78,12 @@ public class AzureCheckpointManager implements CheckpointManager {
   private final Set<TaskName> taskNames = new HashSet<>();
   private final JsonSerdeV2<Map<String, String>> jsonSerde = new JsonSerdeV2<>();
 
-  AzureCheckpointManager(AzureConfig azureConfig) {
+  AzureCheckpointManager(AzureConfig azureConfig, Option<String> jobName) {
+    if (!jobName.isDefined()) {
+      throw new AzureException("Jobs must have a name to use Azure Checkpoint Manager");
+    }
+    // Remove invalid characters
+    jobTableName = jobName.get().replaceAll(REGEX_TABLE_NAME, "");
     storageConnectionString = azureConfig.getAzureConnectionString();
     azureClient = new AzureClient(storageConnectionString);
   }
@@ -82,7 +92,7 @@ public class AzureCheckpointManager implements CheckpointManager {
   public void start() {
     try {
       // Create the table if it doesn't exist.
-      cloudTable = azureClient.getTableClient().getTableReference(CHECKPOINT_MANAGER_TABLE_NAME);
+      cloudTable = azureClient.getTableClient().getTableReference(jobTableName);
       cloudTable.createIfNotExists();
 
     } catch (URISyntaxException e) {
@@ -115,9 +125,13 @@ public void writeCheckpoint(TaskName taskName, Checkpoint checkpoint) {
       SystemStreamPartition ssp = entry.getKey();
       String offset = entry.getValue();
 
+      String partitionKey = taskName.toString();
+      checkValidKey(partitionKey, "Taskname");
+      String rowKey = serializeSystemStreamPartition(ssp);
+      checkValidKey(rowKey, "SystemStreamPartition");
+
       // Create table entity
-      TaskCheckpointEntity taskCheckpoint = new TaskCheckpointEntity(taskName.toString(),
-              serializeSystemStreamPartition(ssp), offset);
+      TaskCheckpointEntity taskCheckpoint = new TaskCheckpointEntity(partitionKey, rowKey, offset);
 
       // Add to batch operation
       batchOperation.insertOrReplace(taskCheckpoint);
@@ -135,6 +149,13 @@ public void writeCheckpoint(TaskName taskName, Checkpoint checkpoint) {
     }
   }
 
+  private void checkValidKey(String key, String fieldUsed) {
+    if (key == null || key.matches(REGEX_INVALID_KEY)) {
+      throw new AzureException(String.format("Cannot insert to Azure Checkpoint Manager; %s %s contains invalid characters [*, /, \\\\, ?]",
+      fieldUsed, key));
+    }
+  }
+
   private String serializeSystemStreamPartition(SystemStreamPartition ssp) {
     // Create the Json string for SystemStreamPartition
     Map<String, String> sspMap = new HashMap<>();
diff --git a/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManagerFactory.java b/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManagerFactory.java
index 3c5d62a120..95728e39e8 100644
--- a/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManagerFactory.java
+++ b/samza-azure/src/main/java/org/apache/samza/checkpoint/azure/AzureCheckpointManagerFactory.java
@@ -23,11 +23,13 @@
 import org.apache.samza.checkpoint.CheckpointManagerFactory;
 import org.apache.samza.config.AzureConfig;
 import org.apache.samza.config.Config;
+import org.apache.samza.config.JobConfig;
 import org.apache.samza.metrics.MetricsRegistry;
 
 public class AzureCheckpointManagerFactory implements CheckpointManagerFactory {
   @Override
   public CheckpointManager getCheckpointManager(Config config, MetricsRegistry registry) {
-    return new AzureCheckpointManager(new AzureConfig(config));
+    JobConfig jobConfig = new JobConfig(config);
+    return new AzureCheckpointManager(new AzureConfig(config), jobConfig.getName());
   }
 }
diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java
index 3bc04f885b..5d8391103e 100644
--- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java
+++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java
@@ -20,10 +20,15 @@
 package org.apache.samza.system.eventhub;
 
 import com.microsoft.azure.eventhubs.EventHubClient;
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
 import org.apache.samza.config.MapConfig;
+import org.apache.samza.config.StreamConfig;
 import org.apache.samza.system.eventhub.producer.EventHubSystemProducer;
+import scala.collection.JavaConversions;
 
 import java.time.Duration;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -47,7 +52,7 @@ public class EventHubConfig extends MapConfig {
           .PartitioningMethod.EVENT_HUB_HASHING.name();
 
   public static final String CONFIG_SEND_KEY_IN_EVENT_PROPERTIES = "systems.%s.eventhubs.send.key";
-  public static final Boolean DEFAULT_CONFIG_SEND_KEY_IN_EVENT_PROPERTIES = false;
+  public static final Boolean DEFAULT_CONFIG_SEND_KEY_IN_EVENT_PROPERTIES = true;
 
   public static final String CONFIG_FETCH_RUNTIME_INFO_TIMEOUT_MILLIS = "systems.%s.eventhubs.runtime.info.timeout";
   public static final long DEFAULT_CONFIG_FETCH_RUNTIME_INFO_TIMEOUT_MILLIS = Duration.ofMinutes(1L).toMillis();
@@ -55,9 +60,47 @@ public class EventHubConfig extends MapConfig {
   public static final String CONFIG_CONSUMER_BUFFER_CAPACITY = "systems.%s.eventhubs.receive.queue.size";
   public static final int DEFAULT_CONFIG_CONSUMER_BUFFER_CAPACITY = 100;
 
+  private final Map<String, String> physcialToId = new HashMap<>();
 
-  public EventHubConfig(Map<String, String> config) {
+  public EventHubConfig(Config config) {
     super(config);
+
+    // Build reverse index for streamName -> streamId
+    StreamConfig streamConfig = new StreamConfig(config);
+    JavaConversions.asJavaCollection(streamConfig.getStreamIds())
+            .forEach((streamId) -> physcialToId.put(streamConfig.getPhysicalName(streamId), streamId));
+  }
+
+  private String getFromStreamIdOrName(String configName, String systemName, String streamName, String defaultString) {
+    String result = getFromStreamIdOrName(configName, systemName, streamName);
+    if (result == null) {
+      return defaultString;
+    }
+    return result;
+  }
+
+  private String getFromStreamIdOrName(String configName, String systemName, String streamName) {
+    String streamId = getStreamId(streamName);
+    return get(String.format(configName, systemName, streamId),
+            streamId.equals(streamName) ? null : get(String.format(configName, systemName, streamName)));
+  }
+
+  private String validateRequiredConfig(String value, String fieldName, String systemName, String streamName) {
+    if (value == null) {
+      throw new SamzaException(String.format("Missing %s configuration for system: %s, stream: %s",
+              fieldName, systemName, streamName));
+    }
+    return value;
+  }
+
+  /**
+   * Get the streamId for the specified streamName
+   *
+   * @param streamName the physical identifier of a stream
+   * @return the streamId identifier for the stream or the queried streamName if it is not found.
+   */
+  public String getStreamId(String streamName) {
+    return physcialToId.getOrDefault(streamName, streamName);
   }
 
   /**
@@ -75,55 +118,59 @@ public List<String> getStreams(String systemName) {
    * Get the EventHubs namespace for the stream
    *
    * @param systemName name of the system
-   * @param streamName name of stream
+   * @param streamName name of stream (physical or streamId)
    * @return EventHubs namespace
    */
   public String getStreamNamespace(String systemName, String streamName) {
-    return get(String.format(CONFIG_STREAM_NAMESPACE, systemName, streamName));
+    return validateRequiredConfig(getFromStreamIdOrName(CONFIG_STREAM_NAMESPACE, systemName, streamName),
+            "Namespace", systemName, streamName);
   }
 
   /**
    * Get the EventHubs entity path (topic name) for the stream
    *
    * @param systemName name of the system
-   * @param streamName name of stream
+   * @param streamName name of stream (physical or streamId)
    * @return EventHubs entity path
    */
   public String getStreamEntityPath(String systemName, String streamName) {
-    return get(String.format(CONFIG_STREAM_ENTITYPATH, systemName, streamName));
+    return validateRequiredConfig(getFromStreamIdOrName(CONFIG_STREAM_ENTITYPATH, systemName, streamName),
+            "EntityPath", systemName, streamName);
   }
 
   /**
    * Get the EventHubs SAS (Shared Access Signature) key name for the stream
    *
    * @param systemName name of the system
-   * @param streamName name of stream
+   * @param streamName name of stream (physical or streamId)
    * @return EventHubs SAS key name
    */
   public String getStreamSasKeyName(String systemName, String streamName) {
-    return get(String.format(CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName));
+    return validateRequiredConfig(getFromStreamIdOrName(CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName),
+            "SASKeyName", systemName, streamName);
   }
 
   /**
    * Get the EventHubs SAS (Shared Access Signature) token for the stream
    *
    * @param systemName name of the system
-   * @param streamName name of stream
+   * @param streamName name of stream (physical or streamId)
    * @return EventHubs SAS token
    */
   public String getStreamSasToken(String systemName, String streamName) {
-    return get(String.format(CONFIG_STREAM_SAS_TOKEN, systemName, streamName));
+    return validateRequiredConfig(getFromStreamIdOrName(CONFIG_STREAM_SAS_TOKEN, systemName, streamName),
+            "SASToken", systemName, streamName);
   }
 
   /**
    * Get the EventHubs consumer group used for consumption for the stream
    *
    * @param systemName name of the system
-   * @param streamName name of stream
+   * @param streamName name of stream (physical or streamId)
    * @return EventHubs consumer group
    */
   public String getStreamConsumerGroup(String systemName, String streamName) {
-    return get(String.format(CONFIG_STREAM_CONSUMER_GROUP, systemName, streamName), DEFAULT_CONFIG_STREAM_CONSUMER_GROUP);
+    return getFromStreamIdOrName(CONFIG_STREAM_CONSUMER_GROUP, systemName, streamName, DEFAULT_CONFIG_STREAM_CONSUMER_GROUP);
   }
 
   /**
diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/SamzaEventHubClientManager.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/SamzaEventHubClientManager.java
index ada5694123..977e25237a 100644
--- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/SamzaEventHubClientManager.java
+++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/SamzaEventHubClientManager.java
@@ -75,8 +75,8 @@ public void init() {
 
       eventHubClient = EventHubClient.createFromConnectionStringSync(connectionStringBuilder.toString(), retryPolicy);
     } catch (IOException | ServiceBusException e) {
-      String msg = String.format("Creation of EventHub client failed for eventHub %s %s %s %s on remote host %s:%d",
-              entityPath, eventHubNamespace, sasKeyName, sasKey, remoteHost, ClientConstants.AMQPS_PORT);
+      String msg = String.format("Creation of EventHub client failed for eventHub EntityPath: %s on remote host %s:%d",
+              entityPath, remoteHost, ClientConstants.AMQPS_PORT);
       LOG.error(msg, e);
       throw new SamzaException(msg, e);
     }
diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/admin/EventHubSystemAdmin.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/admin/EventHubSystemAdmin.java
index 11998a4734..91d504ceb5 100644
--- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/admin/EventHubSystemAdmin.java
+++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/admin/EventHubSystemAdmin.java
@@ -63,8 +63,8 @@ public EventHubSystemAdmin(String systemName, EventHubConfig eventHubConfig,
   private String getNextOffset(String currentOffset) {
     // EventHub will return the first message AFTER the offset
     // that was specified in the fetch request.
-    return currentOffset.equals(EventHubSystemConsumer.END_OF_STREAM) ? currentOffset :
-            String.valueOf(Long.parseLong(currentOffset) + 1);
+    // If no such offset exists Eventhub will return an error.
+    return String.valueOf(Long.parseLong(currentOffset) + 1);
   }
 
   @Override
@@ -158,6 +158,7 @@ private Map<Partition, SystemStreamPartitionMetadata> getPartitionMetadata(Strin
           long timeoutMs = eventHubConfig.getRuntimeInfoWaitTimeMS(systemName);
           EventHubPartitionRuntimeInformation ehPartitionInfo = ehPartitionRuntimeInfo.get(timeoutMs, TimeUnit.MILLISECONDS);
 
+          // Set offsets
           String startingOffset = EventHubSystemConsumer.START_OF_STREAM;
           String newestOffset = ehPartitionInfo.getLastEnqueuedOffset();
           String upcomingOffset = getNextOffset(newestOffset);
@@ -181,12 +182,14 @@ public static Integer compareOffsets(String offset1, String offset2) {
     if (offset1 == null || offset2 == null) {
       return null;
     }
+    // Should NOT be able to compare with END_OF_STREAM to allow new offsets to be
+    // considered caught up if stream started at END_OF_STREAM offset
+    if (EventHubSystemConsumer.END_OF_STREAM.equals(offset1) ||
+            EventHubSystemConsumer.END_OF_STREAM.equals(offset2)) {
+      return null;
+    }
     try {
-      if (offset1.equals(EventHubSystemConsumer.END_OF_STREAM)) {
-        return offset2.equals(EventHubSystemConsumer.END_OF_STREAM) ? 0 : 1;
-      }
-      return offset2.equals(EventHubSystemConsumer.END_OF_STREAM) ? -1 :
-              Long.compare(Long.parseLong(offset1), Long.parseLong(offset2));
+      return Long.compare(Long.parseLong(offset1), Long.parseLong(offset2));
     } catch (NumberFormatException exception) {
       return null;
     }
diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubSystemConsumer.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubSystemConsumer.java
index 4de34de7d8..90c73dc22a 100644
--- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubSystemConsumer.java
+++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubSystemConsumer.java
@@ -20,6 +20,7 @@
 package org.apache.samza.system.eventhub.consumer;
 
 import com.microsoft.azure.eventhubs.EventData;
+import com.microsoft.azure.eventhubs.EventHubPartitionRuntimeInformation;
 import com.microsoft.azure.eventhubs.PartitionReceiveHandler;
 import com.microsoft.azure.eventhubs.PartitionReceiver;
 import com.microsoft.azure.servicebus.ServiceBusException;
@@ -34,6 +35,7 @@
 import org.apache.samza.system.eventhub.Interceptor;
 import org.apache.samza.system.eventhub.admin.EventHubSystemAdmin;
 import org.apache.samza.system.eventhub.metrics.SamzaHistogram;
+import org.apache.samza.system.eventhub.producer.EventHubSystemProducer;
 import org.apache.samza.util.BlockingEnvelopeMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -140,23 +142,23 @@ public EventHubSystemConsumer(EventHubConfig config, String systemName,
     this.config = config;
     this.systemName = systemName;
     this.interceptors = interceptors;
-    List<String> streamNames = config.getStreams(systemName);
+    List<String> streamIds = config.getStreams(systemName);
     // Create and initiate connections to Event Hubs
-    for (String streamName : streamNames) {
+    for (String streamId : streamIds) {
       EventHubClientManager eventHubClientManager = eventHubClientManagerFactory
-              .getEventHubClientManager(systemName, streamName, config);
-      streamEventHubManagers.put(streamName, eventHubClientManager);
+              .getEventHubClientManager(systemName, streamId, config);
+      streamEventHubManagers.put(streamId, eventHubClientManager);
       eventHubClientManager.init();
     }
 
     // Initiate metrics
-    eventReadRates = streamNames.stream()
+    eventReadRates = streamIds.stream()
             .collect(Collectors.toMap(Function.identity(), x -> registry.newCounter(x, EVENT_READ_RATE)));
-    eventByteReadRates = streamNames.stream()
+    eventByteReadRates = streamIds.stream()
             .collect(Collectors.toMap(Function.identity(), x -> registry.newCounter(x, EVENT_BYTE_READ_RATE)));
-    readLatencies = streamNames.stream()
+    readLatencies = streamIds.stream()
             .collect(Collectors.toMap(Function.identity(), x -> new SamzaHistogram(registry, x, READ_LATENCY)));
-    readErrors = streamNames.stream()
+    readErrors = streamIds.stream()
             .collect(Collectors.toMap(Function.identity(), x -> registry.newCounter(x, READ_ERRORS)));
 
     // Locking to ensure that these aggregated metrics will be created only once across multiple system consumers.
@@ -174,20 +176,41 @@ public EventHubSystemConsumer(EventHubConfig config, String systemName,
   public void register(SystemStreamPartition systemStreamPartition, String offset) {
     super.register(systemStreamPartition, offset);
 
+    LOG.info(String.format("Eventhub consumer trying to register ssp %s, offset %s", systemStreamPartition, offset));
     if (isStarted) {
       throw new SamzaException("Trying to add partition when the connection has already started.");
     }
 
     if (streamPartitionOffsets.containsKey(systemStreamPartition)) {
+      // Only update if new offset is lower than previous offset
+      if (END_OF_STREAM.equals(offset)) return;
       String prevOffset = streamPartitionOffsets.get(systemStreamPartition);
-      if (EventHubSystemAdmin.compareOffsets(offset, prevOffset) > -1) {
-        // Only update if new offset is lower than previous offset
+      if (!END_OF_STREAM.equals(prevOffset) && EventHubSystemAdmin.compareOffsets(offset, prevOffset) > -1) {
         return;
       }
     }
     streamPartitionOffsets.put(systemStreamPartition, offset);
   }
 
+  private String getNewestEventHubOffset(EventHubClientManager eventHubClientManager, String streamName, Integer partitionId) {
+    CompletableFuture<EventHubPartitionRuntimeInformation> partitionRuntimeInfoFuture = eventHubClientManager
+            .getEventHubClient()
+            .getPartitionRuntimeInformation(partitionId.toString());
+    try {
+      long timeoutMs = config.getRuntimeInfoWaitTimeMS(systemName);
+
+      EventHubPartitionRuntimeInformation partitionRuntimeInformation = partitionRuntimeInfoFuture
+              .get(timeoutMs, TimeUnit.MILLISECONDS);
+
+      return partitionRuntimeInformation.getLastEnqueuedOffset();
+    } catch (InterruptedException | ExecutionException | TimeoutException e) {
+      String msg = String.format(
+              "Error while fetching EventHubPartitionRuntimeInfo for System:%s, Stream:%s, Partition:%s",
+              systemName, streamName, partitionId);
+      throw new SamzaException(msg);
+    }
+  }
+
   @Override
   public void start() {
     isStarted = true;
@@ -196,27 +219,36 @@ public void start() {
 
       SystemStreamPartition ssp = entry.getKey();
       String streamName = ssp.getStream();
+      String streamId = config.getStreamId(ssp.getStream());
       Integer partitionId = ssp.getPartition().getPartitionId();
       String offset = entry.getValue();
-      String consumerGroup = config.getStreamConsumerGroup(systemName, streamName);
-      String namespace = config.getStreamNamespace(systemName, streamName);
-      String entityPath = config.getStreamEntityPath(systemName, streamName);
-      EventHubClientManager eventHubClientManager = streamEventHubManagers.get(streamName);
+      String consumerGroup = config.getStreamConsumerGroup(systemName, streamId);
+      String namespace = config.getStreamNamespace(systemName, streamId);
+      String entityPath = config.getStreamEntityPath(systemName, streamId);
+      EventHubClientManager eventHubClientManager = streamEventHubManagers.get(streamId);
 
       try {
+        // Fetch the newest offset
+        String newestEventHubOffset = getNewestEventHubOffset(eventHubClientManager, streamName, partitionId);
         PartitionReceiver receiver;
-        if (offset.equals(EventHubSystemConsumer.END_OF_STREAM)) {
+        if (END_OF_STREAM.equals(offset) || EventHubSystemAdmin.compareOffsets(newestEventHubOffset, offset) == -1) {
+          // If the offset is greater than the newest offset, use the use current Instant as
+          // offset to fetch in Eventhub.
           receiver = eventHubClientManager.getEventHubClient()
                   .createReceiverSync(consumerGroup, partitionId.toString(), Instant.now());
         } else {
+          // If the offset is less or equal to the newest offset in the system, it can be
+          // used as the starting offset to receive from. EventHub will return the first
+          // message AFTER the offset that was specified in the fetch request.
+          // If no such offset exists Eventhub will return an error.
           receiver = eventHubClientManager.getEventHubClient()
                   .createReceiverSync(consumerGroup, partitionId.toString(), offset,
                           !offset.equals(EventHubSystemConsumer.START_OF_STREAM));
         }
 
-        PartitionReceiveHandler handler = new PartitionReceiverHandlerImpl(ssp, eventReadRates.get(streamName),
-                eventByteReadRates.get(streamName), readLatencies.get(streamName), readErrors.get(streamName),
-                interceptors.getOrDefault(streamName, null));
+        PartitionReceiveHandler handler = new PartitionReceiverHandlerImpl(ssp, eventReadRates.get(streamId),
+                eventByteReadRates.get(streamId), readLatencies.get(streamId), readErrors.get(streamId),
+                interceptors.getOrDefault(streamId, null));
 
 
         // Timeout for EventHubClient receive
@@ -261,11 +293,11 @@ public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<System
   }
 
   private void renewPartitionReceiver(SystemStreamPartition ssp) {
-
-    EventHubClientManager eventHubClientManager = streamEventHubManagers.get(ssp.getStream());
+    String streamId = config.getStreamId(ssp.getStream());
+    EventHubClientManager eventHubClientManager = streamEventHubManagers.get(streamId);
     String offset = streamPartitionOffsets.get(ssp);
     Integer partitionId = ssp.getPartition().getPartitionId();
-    String consumerGroup = config.getStreamConsumerGroup(ssp.getSystem(), ssp.getStream());
+    String consumerGroup = config.getStreamConsumerGroup(ssp.getSystem(), streamId);
 
     try {
       // Close current receiver
@@ -346,6 +378,9 @@ public void onReceive(Iterable<EventData> events) {
             }
             String offset = event.getSystemProperties().getOffset();
             Object partitionKey = event.getSystemProperties().getPartitionKey();
+            if (partitionKey == null) {
+              partitionKey = event.getProperties().get(EventHubSystemProducer.KEY);
+            }
             try {
               updateMetrics(event);
 
diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/EventHubSystemProducer.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/EventHubSystemProducer.java
index 505421cb03..f294751714 100644
--- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/EventHubSystemProducer.java
+++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/EventHubSystemProducer.java
@@ -58,6 +58,7 @@ public class EventHubSystemProducer implements SystemProducer {
   private static final long DEFAULT_FLUSH_TIMEOUT_MILLIS = Duration.ofMinutes(1L).toMillis();
 
   public static final String PRODUCE_TIMESTAMP = "produce-timestamp";
+  public static final String KEY = "key";
 
   // Metrics recording
   private static final String EVENT_WRITE_RATE = "eventWriteRate";
@@ -75,8 +76,9 @@ public class EventHubSystemProducer implements SystemProducer {
   private static final Object AGGREGATE_METRICS_LOCK = new Object();
 
   public enum PartitioningMethod {
+    ROUND_ROBIN,
     EVENT_HUB_HASHING,
-    PARTITION_KEY_AS_PARTITION,
+    PARTITION_KEY_AS_PARTITION
   }
 
   private final HashMap<String, Counter> eventWriteRate = new HashMap<>();
@@ -85,7 +87,6 @@ public enum PartitioningMethod {
   private final HashMap<String, SamzaHistogram> sendCallbackLatency = new HashMap<>();
   private final HashMap<String, Counter> sendErrors = new HashMap<>();
 
-  private final EventHubClientManagerFactory eventHubClientManagerFactory;
   private final EventHubConfig config;
   private final MetricsRegistry registry;
   private final PartitioningMethod partitioningMethod;
@@ -109,36 +110,35 @@ public EventHubSystemProducer(EventHubConfig config, String systemName,
     this.registry = registry;
     this.systemName = systemName;
     this.partitioningMethod = config.getPartitioningMethod(systemName);
-    this.eventHubClientManagerFactory = eventHubClientManagerFactory;
     this.interceptors = interceptors;
+
+    // Fetches the stream ids
+    List<String> streamIds = config.getStreams(systemName);
+
+    // Create and initiate connections to Event Hubs
+    for (String streamId : streamIds) {
+      EventHubClientManager ehClient = eventHubClientManagerFactory.getEventHubClientManager(systemName, streamId, config);
+      eventHubClients.put(streamId, ehClient);
+      ehClient.init();
+    }
   }
 
   @Override
-  public synchronized void register(String streamName) {
-    LOG.debug("Trying to register {}.", streamName);
+  public synchronized void register(String source) {
     if (isStarted) {
       String msg = "Cannot register once the producer is started.";
       throw new SamzaException(msg);
     }
-
-    if (eventHubClients.containsKey(streamName)) {
-      LOG.warn("Already registered stream {}.", streamName);
-      return;
-    }
-
-    EventHubClientManager ehClient = eventHubClientManagerFactory.getEventHubClientManager(systemName, streamName, config);
-
-    ehClient.init();
-    eventHubClients.put(streamName, ehClient);
   }
 
   @Override
   public synchronized void start() {
     LOG.debug("Starting system producer.");
 
+    // Create partition senders if required
     if (PartitioningMethod.PARTITION_KEY_AS_PARTITION.equals(partitioningMethod)) {
       // Create all partition senders
-      eventHubClients.forEach((streamName, samzaEventHubClient) -> {
+      eventHubClients.forEach((streamId, samzaEventHubClient) -> {
           EventHubClient ehClient = samzaEventHubClient.getEventHubClient();
 
           try {
@@ -153,7 +153,7 @@ public synchronized void start() {
               partitionSenders.put(i, partitionSender);
             }
 
-            streamPartitionSenders.put(streamName, partitionSenders);
+            streamPartitionSenders.put(streamId, partitionSenders);
           } catch (InterruptedException | ExecutionException | TimeoutException e) {
             String msg = "Failed to fetch number of Event Hub partitions for partition sender creation";
             throw new SamzaException(msg, e);
@@ -164,15 +164,16 @@ public synchronized void start() {
         });
     }
 
-    for (String eventHub : eventHubClients.keySet()) {
-      eventWriteRate.put(eventHub, registry.newCounter(eventHub, EVENT_WRITE_RATE));
-      eventByteWriteRate.put(eventHub, registry.newCounter(eventHub, EVENT_BYTE_WRITE_RATE));
-      sendLatency.put(eventHub, new SamzaHistogram(registry, eventHub, SEND_LATENCY));
-      sendCallbackLatency.put(eventHub, new SamzaHistogram(registry, eventHub, SEND_CALLBACK_LATENCY));
-      sendErrors.put(eventHub, registry.newCounter(eventHub, SEND_ERRORS));
-    }
+    // Initiate metrics
+    eventHubClients.keySet().forEach((streamId) -> {
+        eventWriteRate.put(streamId, registry.newCounter(streamId, EVENT_WRITE_RATE));
+        eventByteWriteRate.put(streamId, registry.newCounter(streamId, EVENT_BYTE_WRITE_RATE));
+        sendLatency.put(streamId, new SamzaHistogram(registry, streamId, SEND_LATENCY));
+        sendCallbackLatency.put(streamId, new SamzaHistogram(registry, streamId, SEND_CALLBACK_LATENCY));
+        sendErrors.put(streamId, registry.newCounter(streamId, SEND_ERRORS));
+      });
 
-    // Locking to ensure that these aggregated metrics will be created only once across multiple system consumers.
+    // Locking to ensure that these aggregated metrics will be created only once across multiple system producers.
     synchronized (AGGREGATE_METRICS_LOCK) {
       if (aggEventWriteRate == null) {
         aggEventWriteRate = registry.newCounter(AGGREGATE, EVENT_WRITE_RATE);
@@ -187,35 +188,38 @@ public synchronized void start() {
   }
 
   @Override
-  public synchronized void send(String destination, OutgoingMessageEnvelope envelope) {
+  public synchronized void send(String source, OutgoingMessageEnvelope envelope) {
+    LOG.debug(String.format("Trying to send %s", envelope));
     if (!isStarted) {
       throw new SamzaException("Trying to call send before the producer is started.");
     }
 
-    if (!eventHubClients.containsKey(destination)) {
-      String msg = String.format("Trying to send event to a destination {%s} that is not registered.", destination);
+    String streamId = config.getStreamId(envelope.getSystemStream().getStream());
+
+    if (!eventHubClients.containsKey(streamId)) {
+      String msg = String.format("Trying to send event to a destination {%s} that is not registered.", streamId);
       throw new SamzaException(msg);
     }
 
     checkCallbackThrowable("Received exception on message send");
 
-    EventData eventData = createEventData(destination, envelope);
+    EventData eventData = createEventData(streamId, envelope);
     int eventDataLength =  eventData.getBytes() == null ? 0 : eventData.getBytes().length;
-    eventWriteRate.get(destination).inc();
+    eventWriteRate.get(streamId).inc();
     aggEventWriteRate.inc();
-    eventByteWriteRate.get(destination).inc(eventDataLength);
+    eventByteWriteRate.get(streamId).inc(eventDataLength);
     aggEventByteWriteRate.inc(eventDataLength);
-    EventHubClientManager ehClient = eventHubClients.get(destination);
+    EventHubClientManager ehClient = eventHubClients.get(streamId);
 
     long beforeSendTimeMs = System.currentTimeMillis();
 
     // Async send call
-    CompletableFuture<Void> sendResult = sendToEventHub(destination, eventData, getEnvelopePartitionId(envelope),
+    CompletableFuture<Void> sendResult = sendToEventHub(streamId, eventData, getEnvelopePartitionId(envelope),
             ehClient.getEventHubClient());
 
     long afterSendTimeMs = System.currentTimeMillis();
     long latencyMs = afterSendTimeMs - beforeSendTimeMs;
-    sendLatency.get(destination).update(latencyMs);
+    sendLatency.get(streamId).update(latencyMs);
     aggSendLatency.update(latencyMs);
 
     pendingFutures.add(sendResult);
@@ -223,32 +227,37 @@ public synchronized void send(String destination, OutgoingMessageEnvelope envelo
     // Auto update the metrics and possible throwable when futures are complete.
     sendResult.handle((aVoid, throwable) -> {
         long callbackLatencyMs = System.currentTimeMillis() - afterSendTimeMs;
-        sendCallbackLatency.get(destination).update(callbackLatencyMs);
+        sendCallbackLatency.get(streamId).update(callbackLatencyMs);
         aggSendCallbackLatency.update(callbackLatencyMs);
         if (throwable != null) {
-          sendErrors.get(destination).inc();
+          sendErrors.get(streamId).inc();
           aggSendErrors.inc();
-          LOG.error("Send message to event hub: {} failed with exception: ", destination, throwable);
+          LOG.error("Send message to event hub: {} failed with exception: ", streamId, throwable);
           sendExceptionOnCallback.compareAndSet(null, throwable);
         }
         return aVoid;
       });
   }
 
-  private CompletableFuture<Void> sendToEventHub(String streamName, EventData eventData, Object partitionKey,
+  private CompletableFuture<Void> sendToEventHub(String streamId, EventData eventData, Object partitionKey,
                                                  EventHubClient eventHubClient) {
-    if (partitioningMethod == PartitioningMethod.EVENT_HUB_HASHING) {
+    if (PartitioningMethod.ROUND_ROBIN.equals(partitioningMethod)) {
+      return eventHubClient.send(eventData);
+    } else if (PartitioningMethod.EVENT_HUB_HASHING.equals(partitioningMethod)) {
+      if (partitionKey == null) {
+        throw new SamzaException("Partition key cannot be null for EventHub hashing");
+      }
       return eventHubClient.send(eventData, convertPartitionKeyToString(partitionKey));
-    } else if (partitioningMethod == PartitioningMethod.PARTITION_KEY_AS_PARTITION) {
+    } else if (PartitioningMethod.PARTITION_KEY_AS_PARTITION.equals(partitioningMethod)) {
       if (!(partitionKey instanceof Integer)) {
         String msg = "Partition key should be of type Integer";
         throw new SamzaException(msg);
       }
 
-      Integer numPartition = streamPartitionSenders.get(streamName).size();
+      Integer numPartition = streamPartitionSenders.get(streamId).size();
       Integer destinationPartition = (Integer) partitionKey % numPartition;
 
-      PartitionSender sender = streamPartitionSenders.get(streamName).get(destinationPartition);
+      PartitionSender sender = streamPartitionSenders.get(streamId).get(destinationPartition);
       return sender.send(eventData);
     } else {
       throw new SamzaException("Unknown partitioning method " + partitioningMethod);
@@ -256,7 +265,7 @@ private CompletableFuture<Void> sendToEventHub(String streamName, EventData even
   }
 
   protected Object getEnvelopePartitionId(OutgoingMessageEnvelope envelope) {
-    return envelope.getPartitionKey();
+    return envelope.getPartitionKey() == null ? envelope.getKey() : envelope.getPartitionKey();
   }
 
   private String convertPartitionKeyToString(Object partitionKey) {
@@ -267,12 +276,12 @@ private String convertPartitionKeyToString(Object partitionKey) {
     } else if (partitionKey instanceof byte[]) {
       return new String((byte[]) partitionKey, Charset.defaultCharset());
     } else {
-      throw new SamzaException("Unsupported key type: " + partitionKey.getClass().toString());
+      throw new SamzaException("Unsupported key type: " +  partitionKey.getClass().toString());
     }
   }
 
-  protected EventData createEventData(String streamName, OutgoingMessageEnvelope envelope) {
-    Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamName, null));
+  protected EventData createEventData(String streamId, OutgoingMessageEnvelope envelope) {
+    Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamId, null));
     byte[] eventValue = (byte[]) envelope.getMessage();
     if (interceptor.isPresent()) {
       eventValue = interceptor.get().intercept(eventValue);
@@ -288,7 +297,7 @@ protected EventData createEventData(String streamName, OutgoingMessageEnvelope e
         keyValue = (envelope.getKey() instanceof byte[]) ? new String((byte[]) envelope.getKey())
                 : envelope.getKey().toString();
       }
-      eventData.getProperties().put("key", keyValue);
+      eventData.getProperties().put(KEY, keyValue);
     }
     return eventData;
   }
diff --git a/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java b/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java
index 3e5ead008f..4560b114d1 100644
--- a/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java
+++ b/samza-azure/src/test/java/org/apache/samza/checkpoint/azure/ITestAzureCheckpointManager.java
@@ -34,7 +34,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-@Ignore("Intergration Test")
+@Ignore("Requires Azure account credentials")
 public class ITestAzureCheckpointManager {
 
   private static String storageConnectionString = "";
diff --git a/samza-azure/src/test/java/org/apache/samza/system/eventhub/MockEventHubClientManagerFactory.java b/samza-azure/src/test/java/org/apache/samza/system/eventhub/MockEventHubClientManagerFactory.java
index 1f06f7d310..368087a952 100644
--- a/samza-azure/src/test/java/org/apache/samza/system/eventhub/MockEventHubClientManagerFactory.java
+++ b/samza-azure/src/test/java/org/apache/samza/system/eventhub/MockEventHubClientManagerFactory.java
@@ -101,6 +101,10 @@ private class MockEventHubClientManager implements EventHubClientManager {
           }
           return null;
         });
+      EventHubPartitionRuntimeInformation mockPartitionRuntimeInfo = PowerMockito.mock(EventHubPartitionRuntimeInformation.class);
+      PowerMockito.when(mockPartitionRuntimeInfo.getLastEnqueuedOffset())
+              .thenReturn(EventHubSystemConsumer.START_OF_STREAM);
+      CompletableFuture<EventHubPartitionRuntimeInformation> partitionFuture =  new MockPartitionFuture(mockPartitionRuntimeInfo);
 
       // Producer mocks
       PartitionSender mockPartitionSender0 = PowerMockito.mock(PartitionSender.class);
@@ -137,6 +141,7 @@ private class MockEventHubClientManager implements EventHubClientManager {
                     startingOffsets.put(partitionId, offset);
                     return mockPartitionReceiver;
                   });
+        PowerMockito.when(mockEventHubClient.getPartitionRuntimeInformation(anyString())).thenReturn(partitionFuture);
 
         // Producer calls
         PowerMockito.when(mockEventHubClient.createPartitionSenderSync("0")).thenReturn(mockPartitionSender0);
@@ -191,6 +196,19 @@ public EventHubRuntimeInformation get(long timeout, TimeUnit unit) {
       }
     }
 
+    private class MockPartitionFuture extends CompletableFuture<EventHubPartitionRuntimeInformation> {
+      EventHubPartitionRuntimeInformation runtimeInformation;
+
+      MockPartitionFuture(EventHubPartitionRuntimeInformation runtimeInformation) {
+        this.runtimeInformation = runtimeInformation;
+      }
+
+      @Override
+      public EventHubPartitionRuntimeInformation get(long timeout, TimeUnit unit) {
+        return runtimeInformation;
+      }
+    }
+
   }
 
 }
diff --git a/samza-azure/src/test/java/org/apache/samza/system/eventhub/admin/TestEventHubSystemAdmin.java b/samza-azure/src/test/java/org/apache/samza/system/eventhub/admin/TestEventHubSystemAdmin.java
index c49e68eb70..8861152568 100644
--- a/samza-azure/src/test/java/org/apache/samza/system/eventhub/admin/TestEventHubSystemAdmin.java
+++ b/samza-azure/src/test/java/org/apache/samza/system/eventhub/admin/TestEventHubSystemAdmin.java
@@ -49,14 +49,8 @@ public void testOffsetComparison() {
     Assert.assertEquals(0, eventHubSystemAdmin.offsetComparator("150", "150").intValue());
     Assert.assertEquals(1, eventHubSystemAdmin.offsetComparator("200", "100").intValue());
     Assert.assertNull(eventHubSystemAdmin.offsetComparator("1", "a"));
-    Assert.assertEquals(-1, eventHubSystemAdmin
-            .offsetComparator("100", EventHubSystemConsumer.END_OF_STREAM).intValue());
-    Assert.assertEquals(0, eventHubSystemAdmin.offsetComparator(EventHubSystemConsumer.END_OF_STREAM,
-            EventHubSystemConsumer.END_OF_STREAM).intValue());
-    Assert.assertEquals(1, eventHubSystemAdmin
-            .offsetComparator(EventHubSystemConsumer.END_OF_STREAM, "100").intValue());
-    Assert.assertEquals(-1, eventHubSystemAdmin
-            .offsetComparator(EventHubSystemConsumer.START_OF_STREAM, "10").intValue());
+    Assert.assertNull(eventHubSystemAdmin.offsetComparator("100", EventHubSystemConsumer.END_OF_STREAM));
+    Assert.assertNull(eventHubSystemAdmin.offsetComparator(EventHubSystemConsumer.END_OF_STREAM, EventHubSystemConsumer.END_OF_STREAM));
   }
 
   @Test
@@ -66,16 +60,13 @@ public void testGetNextOffset() {
             MockEventHubConfigFactory.getEventHubConfig(EventHubSystemProducer.PartitioningMethod.EVENT_HUB_HASHING));
     Map<SystemStreamPartition, String> offsets = new HashMap<>();
     SystemStreamPartition ssp0 = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(0));
-    SystemStreamPartition ssp1 = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(1));
     SystemStreamPartition ssp2 = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(2));
     offsets.put(ssp0, Integer.toString(0));
-    offsets.put(ssp1, EventHubSystemConsumer.END_OF_STREAM);
     offsets.put(ssp2, EventHubSystemConsumer.START_OF_STREAM);
 
     Map<SystemStreamPartition, String> updatedOffsets = eventHubSystemAdmin.getOffsetsAfter(offsets);
     Assert.assertEquals(offsets.size(), updatedOffsets.size());
     Assert.assertEquals("1", updatedOffsets.get(ssp0));
-    Assert.assertEquals("-2", updatedOffsets.get(ssp1));
     Assert.assertEquals("0", updatedOffsets.get(ssp2));
   }
 
@@ -102,8 +93,6 @@ public void testGetStreamMetadata() {
       partitionMetadataMap.forEach((partition, metadata) -> {
           Assert.assertEquals(EventHubSystemConsumer.START_OF_STREAM, metadata.getOldestOffset());
           Assert.assertNotSame(EventHubSystemConsumer.END_OF_STREAM, metadata.getNewestOffset());
-          Assert.assertTrue(Long.parseLong(EventHubSystemConsumer.END_OF_STREAM)
-                  <= Long.parseLong(metadata.getNewestOffset()));
           String expectedUpcomingOffset = String.valueOf(Long.parseLong(metadata.getNewestOffset()) + 1);
           Assert.assertEquals(expectedUpcomingOffset, metadata.getUpcomingOffset());
         });
diff --git a/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/ITestEventHubSystemConsumer.java b/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/ITestEventHubSystemConsumer.java
index dbf8807f86..cfd82170d9 100644
--- a/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/ITestEventHubSystemConsumer.java
+++ b/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/ITestEventHubSystemConsumer.java
@@ -29,14 +29,15 @@
 import org.apache.samza.system.eventhub.TestMetricsRegistry;
 import org.apache.samza.system.eventhub.producer.EventHubSystemProducer;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 
 import static org.apache.samza.system.eventhub.MockEventHubConfigFactory.STREAM_NAME1;
 import static org.apache.samza.system.eventhub.MockEventHubConfigFactory.SYSTEM_NAME;
 
+@Ignore("Requires Azure account credentials")
 public class ITestEventHubSystemConsumer {
 
   private Config createEventHubConfig() {
diff --git a/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/TestEventHubSystemConsumer.java b/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/TestEventHubSystemConsumer.java
index a25a3b61e7..865a248fc5 100644
--- a/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/TestEventHubSystemConsumer.java
+++ b/samza-azure/src/test/java/org/apache/samza/system/eventhub/consumer/TestEventHubSystemConsumer.java
@@ -22,6 +22,7 @@
 
 import com.microsoft.azure.eventhubs.*;
 import org.apache.samza.Partition;
+import org.apache.samza.config.MapConfig;
 import org.apache.samza.metrics.Counter;
 import org.apache.samza.system.IncomingMessageEnvelope;
 import org.apache.samza.system.SystemStreamPartition;
@@ -41,7 +42,8 @@
 import static org.apache.samza.system.eventhub.MockEventHubConfigFactory.*;
 
 @RunWith(PowerMockRunner.class)
-@PrepareForTest({EventHubRuntimeInformation.class, EventHubClient.class, PartitionReceiver.class, PartitionSender.class})
+@PrepareForTest({EventHubRuntimeInformation.class, EventHubPartitionRuntimeInformation.class,
+        EventHubClient.class, PartitionReceiver.class, PartitionSender.class})
 public class TestEventHubSystemConsumer {
   private static final String MOCK_ENTITY_1 = "mocktopic1";
   private static final String MOCK_ENTITY_2 = "mocktopic2";
@@ -85,11 +87,12 @@ public void testMultipleRegistersToSameSSP() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName), EVENTHUB_KEY_NAME);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, systemName, streamName), EVENTHUB_KEY);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, systemName, streamName), MOCK_ENTITY_1);
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory eventHubClientWrapperFactory = new MockEventHubClientManagerFactory(eventData);
 
     EventHubSystemConsumer consumer =
-            new EventHubSystemConsumer(new EventHubConfig(configMap), systemName, eventHubClientWrapperFactory, interceptors,
+            new EventHubSystemConsumer(new EventHubConfig(config), systemName, eventHubClientWrapperFactory, interceptors,
                     testMetrics);
     consumer.register(ssp, "1");
     consumer.register(ssp, EventHubSystemConsumer.END_OF_STREAM);
@@ -124,11 +127,12 @@ public void testSinglePartitionConsumptionHappyPath() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName), EVENTHUB_KEY_NAME);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, systemName, streamName), EVENTHUB_KEY);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, systemName, streamName), MOCK_ENTITY_1);
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory eventHubClientWrapperFactory = new MockEventHubClientManagerFactory(eventData);
 
     EventHubSystemConsumer consumer =
-            new EventHubSystemConsumer(new EventHubConfig(configMap), systemName, eventHubClientWrapperFactory, interceptors,
+            new EventHubSystemConsumer(new EventHubConfig(config), systemName, eventHubClientWrapperFactory, interceptors,
                     testMetrics);
     consumer.register(ssp, EventHubSystemConsumer.END_OF_STREAM);
     consumer.start();
@@ -173,11 +177,12 @@ public void testSinglePartitionConsumptionInterceptor() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName), EVENTHUB_KEY_NAME);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, systemName, streamName), EVENTHUB_KEY);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, systemName, streamName), MOCK_ENTITY_1);
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory eventHubClientWrapperFactory = new MockEventHubClientManagerFactory(eventData);
 
     EventHubSystemConsumer consumer =
-            new EventHubSystemConsumer(new EventHubConfig(configMap), systemName, eventHubClientWrapperFactory, interceptors,
+            new EventHubSystemConsumer(new EventHubConfig(config), systemName, eventHubClientWrapperFactory, interceptors,
                     testMetrics);
     consumer.register(ssp, EventHubSystemConsumer.END_OF_STREAM);
     consumer.start();
@@ -224,11 +229,12 @@ public void testMultiPartitionConsumptionHappyPath() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, systemName, streamName), EVENTHUB_NAMESPACE);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName), EVENTHUB_KEY_NAME);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, systemName, streamName), EVENTHUB_KEY);
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory eventHubClientWrapperFactory = new MockEventHubClientManagerFactory(eventData);
 
     EventHubSystemConsumer consumer =
-            new EventHubSystemConsumer(new EventHubConfig(configMap), systemName, eventHubClientWrapperFactory, interceptor,
+            new EventHubSystemConsumer(new EventHubConfig(config), systemName, eventHubClientWrapperFactory, interceptor,
                     testMetrics);
     consumer.register(ssp1, EventHubSystemConsumer.START_OF_STREAM);
     consumer.register(ssp2, EventHubSystemConsumer.START_OF_STREAM);
@@ -284,11 +290,12 @@ public void testMultiStreamsConsumptionHappyPath() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, systemName, streamName2), EVENTHUB_NAMESPACE);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName2), EVENTHUB_KEY_NAME);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, systemName, streamName2), EVENTHUB_KEY);
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory eventHubClientWrapperFactory = new MockEventHubClientManagerFactory(eventData);
 
     EventHubSystemConsumer consumer =
-            new EventHubSystemConsumer(new EventHubConfig(configMap), systemName, eventHubClientWrapperFactory, interceptor,
+            new EventHubSystemConsumer(new EventHubConfig(config), systemName, eventHubClientWrapperFactory, interceptor,
                     testMetrics);
 
     consumer.register(ssp1, EventHubSystemConsumer.START_OF_STREAM);
diff --git a/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/ITestEventHubSystemProducer.java b/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/ITestEventHubSystemProducer.java
index cc401984ce..32b1604ad3 100644
--- a/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/ITestEventHubSystemProducer.java
+++ b/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/ITestEventHubSystemProducer.java
@@ -30,12 +30,14 @@
 import org.apache.samza.system.eventhub.consumer.EventHubSystemConsumer;
 import org.apache.samza.util.NoOpMetricsRegistry;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static org.apache.samza.system.eventhub.MockEventHubConfigFactory.*;
 
+@Ignore("Requires Azure account credentials")
 public class ITestEventHubSystemProducer {
   private static final Logger LOG = LoggerFactory.getLogger(ITestEventHubSystemProducer.class.getName());
 
diff --git a/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/TestEventHubSystemProducer.java b/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/TestEventHubSystemProducer.java
index 10016ecea5..ef73775efc 100644
--- a/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/TestEventHubSystemProducer.java
+++ b/samza-azure/src/test/java/org/apache/samza/system/eventhub/producer/TestEventHubSystemProducer.java
@@ -20,6 +20,7 @@
 package org.apache.samza.system.eventhub.producer;
 
 import com.microsoft.azure.eventhubs.*;
+import org.apache.samza.config.MapConfig;
 import org.apache.samza.system.OutgoingMessageEnvelope;
 import org.apache.samza.system.SystemStream;
 import org.apache.samza.system.eventhub.EventHubConfig;
@@ -40,9 +41,12 @@
 import static org.apache.samza.system.eventhub.MockEventHubConfigFactory.*;
 
 @RunWith(PowerMockRunner.class)
-@PrepareForTest({EventHubRuntimeInformation.class, EventHubClient.class, PartitionReceiver.class, PartitionSender.class})
+@PrepareForTest({EventHubRuntimeInformation.class, EventHubPartitionRuntimeInformation.class,
+        EventHubClient.class, PartitionReceiver.class, PartitionSender.class})
 public class TestEventHubSystemProducer {
 
+  private static final String SOURCE = "TestEventHubSystemProducer";
+
   private static List<String> generateMessages(int numMsg) {
     Random rand = new Random(System.currentTimeMillis());
     List<String> messages = new ArrayList<>();
@@ -76,20 +80,21 @@ public void testSendingToSpecificPartitions() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, systemName, streamName), EVENTHUB_ENTITY1);
     configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName),
             PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
 
     EventHubSystemProducer producer =
-            new EventHubSystemProducer(new EventHubConfig(configMap), systemName, factory, interceptor, testMetrics);
+            new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
 
     SystemStream systemStream = new SystemStream(systemName, streamName);
-    producer.register(streamName);
+    producer.register(SOURCE);
     producer.start();
 
     outgoingMessagesP0.forEach(message ->
-            producer.send(streamName, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
+            producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
     outgoingMessagesP1.forEach(message ->
-            producer.send(streamName, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
+            producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
 
     // Retrieve sent data
     List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0)
@@ -126,20 +131,21 @@ public void testSendingToSpecificPartitionsWithInterceptor() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, systemName, streamName), EVENTHUB_ENTITY1);
     configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName),
             PartitioningMethod.PARTITION_KEY_AS_PARTITION.toString());
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
 
     EventHubSystemProducer producer =
-            new EventHubSystemProducer(new EventHubConfig(configMap), systemName, factory, interceptors, testMetrics);
+            new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptors, testMetrics);
 
     SystemStream systemStream = new SystemStream(systemName, streamName);
-    producer.register(streamName);
+    producer.register(SOURCE);
     producer.start();
 
     outgoingMessagesP0.forEach(message ->
-            producer.send(streamName, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
+            producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
     outgoingMessagesP1.forEach(message ->
-            producer.send(streamName, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
+            producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
 
     // Retrieve sent data
     List<String> receivedData0 = factory.getSentData(systemName, streamName, partitionId0)
@@ -180,23 +186,25 @@ public void testSendingToEventHubHashing() throws Exception {
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, systemName, streamName), EVENTHUB_KEY_NAME);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, systemName, streamName), EVENTHUB_KEY);
     configMap.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, systemName, streamName), EVENTHUB_ENTITY1);
+
     // mod 2 on the partitionid to simulate consistent hashing
     configMap.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName),
             PartitioningMethod.EVENT_HUB_HASHING.toString());
+    MapConfig config = new MapConfig(configMap);
 
     MockEventHubClientManagerFactory factory = new MockEventHubClientManagerFactory();
 
     EventHubSystemProducer producer =
-            new EventHubSystemProducer(new EventHubConfig(configMap), systemName, factory, interceptor, testMetrics);
+            new EventHubSystemProducer(new EventHubConfig(config), systemName, factory, interceptor, testMetrics);
 
     SystemStream systemStream = new SystemStream(systemName, streamName);
-    producer.register(streamName);
+    producer.register(SOURCE);
     producer.start();
 
     outgoingMessagesP0.forEach(message ->
-            producer.send(streamName, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
+            producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId0, null, message.getBytes())));
     outgoingMessagesP1.forEach(message ->
-            producer.send(streamName, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
+            producer.send(SOURCE, new OutgoingMessageEnvelope(systemStream, partitionId1, null, message.getBytes())));
 
     // Retrieve sent data
     List<String> receivedData0 = factory.getSentData(systemName, streamName, 0)

From 42bac30ad1832afa92767a5ea492bfd6c3de664a Mon Sep 17 00:00:00 2001
From: Srinivasulu Punuru <spunuru@linkedin.com>
Date: Tue, 12 Dec 2017 10:46:01 -0800
Subject: [PATCH 09/36] Documentation for Samza SQL

**Samza tools** :
Contains the following tools that can be used for playing with Samza sql or any other samza job.

1. Generate kafka events : Tool used to generate avro serialized kafka events
2. Event hub consumer : Tool used to consume events from event hubs topic. This can be used if the samza job writes events to event hubs.
3. Samza sql console : Tool used to execute SQL using samza sql.

Adds documentation on how to use Samza SQL on a local machine and on a yarn environment and their associated Samza tooling.

https://issues.apache.org/jira/browse/SAMZA-1526

Author: Srinivasulu Punuru <spunuru@linkedin.com>

Reviewers: Yi Pan<nickpan47@gmail.com>, Jagadish<jagadish@apachehe.org>

Closes #374 from srinipunuru/docs.1
---
 build.gradle                                  |  28 +++
 docs/README.md                                |   4 +-
 docs/learn/tutorials/versioned/index.md       |   3 +
 docs/learn/tutorials/versioned/samza-sql.md   | 123 +++++++++++
 docs/learn/tutorials/versioned/samza-tools.md | 109 ++++++++++
 docs/startup/download/index.md                |   7 +
 gradle/dependency-versions.gradle             |   1 +
 .../samza/sql/avro/AvroRelConverter.java      |   6 +-
 samza-tools/config/eh-consumer-log4j.xml      |  35 +++
 .../config/generate-kafka-events-log4j.xml    |  35 +++
 .../config/samza-sql-console-log4j.xml        |  35 +++
 samza-tools/scripts/eh-consumer.sh            |  34 +++
 samza-tools/scripts/generate-kafka-events.sh  |  34 +++
 samza-tools/scripts/samza-sql-console.sh      |  34 +++
 .../apache/samza/tools/CommandLineHelper.java |  42 ++++
 .../tools/ConsoleLoggingSystemFactory.java    | 126 +++++++++++
 .../samza/tools/EventHubConsoleConsumer.java  | 120 ++++++++++
 .../samza/tools/GenerateKafkaEvents.java      | 205 ++++++++++++++++++
 .../samza/tools/RandomValueGenerator.java     |  87 ++++++++
 .../apache/samza/tools/SamzaSqlConsole.java   | 188 ++++++++++++++++
 .../tools/avro/AvroSchemaGenRelConverter.java |  94 ++++++++
 .../AvroSchemaGenRelConverterFactory.java     |  43 ++++
 .../samza/tools/avro/AvroSerDeFactory.java    |  96 ++++++++
 .../tools/json/JsonRelConverterFactory.java   |  93 ++++++++
 .../samza/tools/schemas/PageViewEvent.avsc    |  51 +++++
 .../samza/tools/schemas/PageViewEvent.java    |  60 +++++
 .../tools/schemas/ProfileChangeEvent.avsc     |  51 +++++
 .../tools/schemas/ProfileChangeEvent.java     |  60 +++++
 .../apache/samza/tools/udf/RegexMatchUdf.java |  40 ++++
 samza-tools/src/main/resources/log4j.xml      |  43 ++++
 settings.gradle                               |   3 +-
 31 files changed, 1886 insertions(+), 4 deletions(-)
 create mode 100644 docs/learn/tutorials/versioned/samza-sql.md
 create mode 100644 docs/learn/tutorials/versioned/samza-tools.md
 create mode 100644 samza-tools/config/eh-consumer-log4j.xml
 create mode 100644 samza-tools/config/generate-kafka-events-log4j.xml
 create mode 100644 samza-tools/config/samza-sql-console-log4j.xml
 create mode 100755 samza-tools/scripts/eh-consumer.sh
 create mode 100755 samza-tools/scripts/generate-kafka-events.sh
 create mode 100755 samza-tools/scripts/samza-sql-console.sh
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/CommandLineHelper.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/ConsoleLoggingSystemFactory.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/EventHubConsoleConsumer.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/GenerateKafkaEvents.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/RandomValueGenerator.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverter.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverterFactory.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSerDeFactory.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/json/JsonRelConverterFactory.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.avsc
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.avsc
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.java
 create mode 100644 samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java
 create mode 100644 samza-tools/src/main/resources/log4j.xml

diff --git a/build.gradle b/build.gradle
index 330ff0b5e3..04b42f261d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -323,6 +323,34 @@ project(':samza-sql') {
   }
 }
 
+project(':samza-tools') {
+  apply plugin: 'java'
+
+  dependencies {
+    compile project(':samza-sql')
+    compile project(':samza-api')
+    compile project(':samza-azure')
+    compile "log4j:log4j:$log4jVersion"
+    compile "org.slf4j:slf4j-api:$slf4jVersion"
+    compile "org.slf4j:slf4j-log4j12:$slf4jVersion"
+    compile "commons-cli:commons-cli:$commonsCliVersion"
+    compile "org.apache.avro:avro:$avroVersion"
+    compile "org.apache.commons:commons-lang3:$commonsLang3Version"
+    compile "org.apache.kafka:kafka-clients:$kafkaVersion"
+  }
+
+  tasks.create(name: "releaseToolsTarGz", dependsOn: configurations.archives.artifacts, type: Tar) {
+    into "samza-tools-${version}"
+    compression = Compression.GZIP
+    from(project.file("./scripts")) { into "scripts/" }
+    from(project.file("./config")) { into "config/" }
+    from(project(':samza-shell').file("src/main/bash/run-class.sh")) { into "scripts/" }
+    from(configurations.runtime) { into("lib/") }
+    from(configurations.archives.artifacts.files) { into("lib/") }
+    duplicatesStrategy 'exclude'
+  }
+}
+
 project(":samza-kafka_$scalaVersion") {
   apply plugin: 'scala'
 
diff --git a/docs/README.md b/docs/README.md
index 21a49915fc..3de2a781da 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -83,8 +83,8 @@ Following can be done when updating the gradle.properties file
 
     * if this is a major release, add the x.x.x release to Archive category in docs/_layouts/default.html and x.x.x release part in docs/archive/index.html
 
-    * update the download page to use x.x.x release
-      * docs/startup/download/index.md
+    * update the download page (docs/startup/download/index.md) to use x.x.x release
+      * Add an entry to the Sources releases and Samza Tools section to use the new x.x.x release
 
     * update the version number in "tar -xvf ./target/hello-samza-y.y.y-dist.tar.gz -C deploy/samza" in each of the tutorials (and search for other uses of version x.x.x which may need to be replaced with y.y.y)
       * docs/startup/hello-samza/versioned/index.md
diff --git a/docs/learn/tutorials/versioned/index.md b/docs/learn/tutorials/versioned/index.md
index a9ac6a7441..d155b82076 100644
--- a/docs/learn/tutorials/versioned/index.md
+++ b/docs/learn/tutorials/versioned/index.md
@@ -42,6 +42,9 @@ title: Tutorials
 
 [Samza Async API and Multithreading User Guide](samza-async-user-guide.html)
 
+[Samza SQL User Guide](samza-sql.html)
+
+
 <!-- TODO a bunch of tutorials
 [Log Walkthrough](log-walkthrough.html)
 <a href="configuring-kafka-system.html">Configuring a Kafka System</a><br/>
diff --git a/docs/learn/tutorials/versioned/samza-sql.md b/docs/learn/tutorials/versioned/samza-sql.md
new file mode 100644
index 0000000000..fa79a1bbe2
--- /dev/null
+++ b/docs/learn/tutorials/versioned/samza-sql.md
@@ -0,0 +1,123 @@
+---
+layout: page
+title: How to use Samza SQL
+---
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+There are couple of ways to use Samza SQL
+
+1. Run Samza SQL on your local machine.
+2. Run Samza SQL on YARN.
+
+# Running Samza SQL on your local machine  
+
+
+Samza SQL console tool documented [here](samza-tools.html) uses Samza standalone to run the Samza SQL on your local machine. This is the quickest way to play with Samza SQL. Please follow the instructions [here](samza-tools.html) to get access to the Samza tools on your machine.
+
+## Start the Kafka server
+
+Please follow the instructions from the [Kafka quickstart](http://kafka.apache.org/quickstart) to start the zookeeper and Kafka server.
+
+## Create ProfileChangeStream Kafka topic
+
+The below sql statements requires a topic named ProfileChangeStream to be created on the Kafka broker. You can follow the instructions in the [Kafka quick start guide](http://kafka.apache.org/quickstart) to create a topic named "ProfileChangeStream".
+
+{% highlight bash %}
+./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic ProfileChangeStream
+{% endhighlight %}
+
+## Generate events into ProfileChangeStream topic
+
+Use generate-kafka-events from [Samza tools](samza-tools.html) to generate events into the ProfileChangeStream
+
+{% highlight bash %}
+cd samza-tools-<version>
+./scripts/generate-kafka-events.sh -t ProfileChangeStream -e ProfileChange
+{% endhighlight %}
+
+## Using Samza SQL Console to run Samza sql on your local machine
+
+Below are some of the sql queries that you can execute using the samza-sql-console tool from [Samza tools](samza-tools.html) package.
+
+{% highlight bash %}
+# This command just prints out all the events in the Kafka topic ProfileChangeStream into console output as a json serialized payload.
+./scripts/samza-sql-console.sh --sql "insert into log.consoleoutput select * from kafka.ProfileChangeStream"
+
+# This command prints out the fields that are selected into the console output as a json serialized payload.
+./scripts/samza-sql-console.sh --sql "insert into log.consoleoutput select Name, OldCompany, NewCompany from kafka.ProfileChangeStream"
+
+# This command showcases the RegexMatch udf and filtering capabilities.
+./scripts/samza-sql-console.sh --sql "insert into log.consoleoutput select Name as __key__, Name, NewCompany, RegexMatch('.*soft', OldCompany) from kafka.ProfileChangeStream where NewCompany = 'LinkedIn'"
+{% endhighlight %}
+
+
+# Running Samza SQL on YARN
+
+The [hello-samza](https://github.com/apache/samza-hello-samza) project is an example project designed to help you run your first Samza application. It has examples of applications using the low level task API, high level API as well as Samza SQL.
+
+This tutorial demonstrates a simple Samza application that uses SQL to perform stream processing.
+
+## Get the hello-samza Code and Start the grid
+
+Please follow the instructions from [hello-samza-high-level-yarn](hello-samza-high-level-yarn.html) on how to build the hello-samza repository and start the yarn grid. 
+
+## Create the topic and generate Kafka events
+
+Please follow the steps in the section "Create ProfileChangeStream Kafka topic" and "Generate events into ProfileChangeStream topic" above.
+
+## Build a Samza Application Package
+
+Before you can run a Samza application, you need to build a package for it. Please follow the instructions from [hello-samza-high-level-yarn](hello-samza-high-level-yarn.html) on how to build the hello-samza application package.
+
+## Run a Samza Application
+
+After you've built your Samza package, you can start the app on the grid using the run-app.sh script.
+
+{% highlight bash %}
+./deploy/samza/bin/run-app.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/page-view-filter-sql.properties
+{% endhighlight %}
+
+The app executes the following SQL command :
+{% highlight sql %}
+insert into kafka.NewLinkedInEmployees select Name from ProfileChangeStream where NewCompany = 'LinkedIn'
+{% endhighlight %}
+
+This SQL performs the following
+
+1. Consumes the Kafka topic ProfileChangeStreamStream which contains the avro serialized ProfileChangeEvent(s) 
+2. Deserializes the events and filters out only the profile change events where NewCompany = 'LinkedIn' i.e. Members who have moved to LinkedIn.
+3. Writes the Avro serialized event that contains the Id and Name of those profiles to Kafka topic NewLinkedInEmployees.
+
+
+Give the job a minute to startup, and then tail the Kafka topic:
+
+{% highlight bash %}
+./deploy/kafka/bin/kafka-console-consumer.sh  --zookeeper localhost:2181 --topic NewLinkedInEmployees
+{% endhighlight %}
+
+
+Congratulations! You've now setup a local grid that includes YARN, Kafka, and ZooKeeper, and run a Samza SQL application on it.
+
+## Shutdown and cleanup
+
+To shutdown the app, use the same _run-app.sh_ script with an extra _--operation=kill_ argument
+{% highlight bash %}
+./deploy/samza/bin/run-app.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/page-view-filter-sql.properties --operation=kill
+{% endhighlight %}
+
+Please follow the instructions from [Hello Samza High Level API - YARN Deployment](hello-samza-high-level-yarn.html) on how to shutdown and cleanup the app.
diff --git a/docs/learn/tutorials/versioned/samza-tools.md b/docs/learn/tutorials/versioned/samza-tools.md
new file mode 100644
index 0000000000..6e4ee71a05
--- /dev/null
+++ b/docs/learn/tutorials/versioned/samza-tools.md
@@ -0,0 +1,109 @@
+---
+layout: page
+title: How to use Samza tools
+---
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+
+# Get Samza tools
+
+Please visit the [Download page] (/startup/download) to download the Samza tools package
+
+{% highlight bash %}
+tar -xvzf samza-tools-*.tgz
+cd samza-tools-<version>
+{% endhighlight %}
+
+
+# Using Samza tools
+
+
+## Generate kafka events
+
+
+Generate kafka events tool is used to insert avro serialized events into kafka topics. Right now it can insert two types of events [PageViewEvent](https://github.com/apache/samza/blob/master/samza-tools/src/main/java/org/com/linkedin/samza/tools/schemas/PageViewEvent.avsc) and [ProfileChangeEvent](https://github.com/apache/samza/blob/master/samza-tools/src/main/java/org/com/linkedin/samza/tools/schemas/ProfileChangeEvent.avsc)
+
+Before you can generate kafka events, Please follow instructions [here](http://kafka.apache.org/quickstart) to start the zookeeper and kafka server on your local machine.
+
+You can follow below instructions on how to use Generate kafka events tool.
+
+{% highlight bash %}
+
+# Usage of the tool
+
+./scripts/generate-kafka-events.sh
+usage: Error: Missing required options: t, e
+              generate-kafka-events.sh
+ -b,--broker <BROKER>               Kafka broker endpoint Default (localhost:9092).
+ -n,--numEvents <NUM_EVENTS>        Number of events to be produced, 
+                                    Default - Produces events continuously every second.
+ -p,--partitions <NUM_PARTITIONS>   Number of partitions in the topic,
+                                    Default (4).
+ -t,--topic <TOPIC_NAME>            Name of the topic to write events to.
+ -e,--eventtype <EVENT_TYPE>        Type of the event values can be (PageView|ProfileChange). 
+
+
+# Example command to generate 100 events of type PageViewEvent into topic named PageViewStream
+
+ ./scripts/generate-kafka-events.sh -t PageViewStream -e PageView -n 100
+
+
+# Example command to generate ProfileChange events continuously into topic named ProfileChangeStream
+
+ ./scripts/generate-kafka-events.sh -t ProfileChangeStream -e ProfileChange 
+
+{% endhighlight %}
+
+## Samza SQL console tool
+
+Once you generated the events into the kafka topic. Now you can use samza-sql-console tool to perform processing on the events published into the kafka topic.
+
+There are two ways to use the tool -
+
+1. You can either pass the sql statement directly as an argument to the tool. 
+2. You can write the sql statement(s) into a file and pass the sql file as an argument to the tool.
+
+Second option allows you to execute multiple sql statements, whereas the first one lets you execute one at a time.
+
+Samza SQL needs all the events in the topic to be uniform schema. And it also needs access to the schema corresponding to the events in a topic. Typically in an organization, there is a deployment of schema registry which maps topics to schemas. 
+
+In the absence of schema registry, Samza SQL console tool uses the convention to identify the schemas associated with the topic. If the topic name has string "page" it assumes the topic has PageViewEvents else ProfileChangeEvents. 
+
+{% highlight bash %}
+
+# Usage of the tool
+
+ ./scripts/samza-sql-console.sh
+usage: Error: One of the (f or s) options needs to be set
+              samza-sql-console.sh
+ -f,--file <SQL_FILE>   Path to the SQL file to execute.
+ -s,--sql <SQL_STMT>    SQL statement to execute.
+
+# Example command to filter out all the users who have moved to LinkedIn
+
+./scripts/samza-sql-console.sh --sql "Insert into log.consoleOutput select Name, OldCompany from kafka.ProfileChangeStream where NewCompany = 'LinkedIn'"
+
+{% endhighlight %}
+
+You can run below sql commands using Samza sql console. Please make sure you are running generate-kafka-events tool to generate events into ProfileChangeStream before running the below command.
+
+{% highlight bash %}
+./scripts/samza-sql-console.sh --sql "Insert into log.consoleOutput select Name, OldCompany from kafka.ProfileChangeStream where NewCompany = 'LinkedIn'"
+
+{% endhighlight %}
diff --git a/docs/startup/download/index.md b/docs/startup/download/index.md
index b39f90fbee..5dcf38022b 100644
--- a/docs/startup/download/index.md
+++ b/docs/startup/download/index.md
@@ -27,6 +27,12 @@ If you just want to play around with Samza for the first time, go to [Hello Samz
 
 Starting from 2016, Samza will begin requiring JDK8 or higher. Please see [this mailing list thread](http://mail-archives.apache.org/mod_mbox/samza-dev/201610.mbox/%3CCAHUevGGnOQD_VmLWEdpFNq3Lv%2B6gQQmw_JKx9jDr5Cw%2BxFfGtQ%40mail.gmail.com%3E) for details on this decision.
 
+### Samza Tools
+ 
+ Samza tools package contains command line tools that user can run to use Samza and it's input/output systems. 
+ 
+ * [samza-tools-0.14.0.tgz](tbd)
+
 ### Source Releases
 
  * [samza-sources-0.13.1.tgz](http://www.apache.org/dyn/closer.lua/samza/0.13.1)
@@ -40,6 +46,7 @@ Starting from 2016, Samza will begin requiring JDK8 or higher. Please see [this
  * [samza-sources-0.8.0-incubating.tgz](https://archive.apache.org/dist/incubator/samza/0.8.0-incubating)
  * [samza-sources-0.7.0-incubating.tgz](https://archive.apache.org/dist/incubator/samza/0.7.0-incubating)
 
+
 ### Maven
 
 All Samza JARs are published through [Apache's Maven repository](https://repository.apache.org/content/groups/public/org/apache/samza/).
diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle
index 4f467ab8d3..20a1d56efd 100644
--- a/gradle/dependency-versions.gradle
+++ b/gradle/dependency-versions.gradle
@@ -20,6 +20,7 @@
   apacheCommonsCollections4Version = "4.0"
   avroVersion = "1.7.0"
   calciteVersion = "1.14.0"
+  commonsCliVersion = "1.2"
   commonsCodecVersion = "1.9"
   commonsCollectionVersion = "3.2.1"
   commonsHttpClientVersion = "3.1"
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java b/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
index 1c172950b4..ab46a9870c 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
@@ -167,7 +167,11 @@ public SamzaSqlRelMessage convertToRelMessage(KV<Object, Object> samzaMessage) {
 
   @Override
   public KV<Object, Object> convertToSamzaMessage(SamzaSqlRelMessage relMessage) {
-    GenericRecord record = new GenericData.Record(this.avroSchema);
+    return convertToSamzaMessage(relMessage, this.avroSchema);
+  }
+
+  protected KV<Object, Object> convertToSamzaMessage(SamzaSqlRelMessage relMessage, Schema avroSchema) {
+    GenericRecord record = new GenericData.Record(avroSchema);
     List<String> fieldNames = relMessage.getFieldNames();
     List<Object> values = relMessage.getFieldValues();
     for (int index = 0; index < fieldNames.size(); index++) {
diff --git a/samza-tools/config/eh-consumer-log4j.xml b/samza-tools/config/eh-consumer-log4j.xml
new file mode 100644
index 0000000000..d971512d39
--- /dev/null
+++ b/samza-tools/config/eh-consumer-log4j.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+-->
+<!DOCTYPE log4j:configuration PUBLIC "-//APACHE//DTD LOG4J 1.2//EN" "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="fileAppender" class="org.apache.log4j.FileAppender">
+    <param name="File"   value="./eh-consumer.log" />
+    <param name="Append" value="false" />
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5p [%c{1}:%L] - %m%n"/>
+    </layout>
+  </appender>
+  <root>
+    <priority value ="info" />
+    <appender-ref ref="fileAppender" />
+  </root>
+</log4j:configuration>
\ No newline at end of file
diff --git a/samza-tools/config/generate-kafka-events-log4j.xml b/samza-tools/config/generate-kafka-events-log4j.xml
new file mode 100644
index 0000000000..98c7a28e76
--- /dev/null
+++ b/samza-tools/config/generate-kafka-events-log4j.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+-->
+<!DOCTYPE log4j:configuration PUBLIC "-//APACHE//DTD LOG4J 1.2//EN" "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="fileAppender" class="org.apache.log4j.FileAppender">
+    <param name="File"   value="./generate-kafka-events.log" />
+    <param name="Append" value="false" />
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5p [%c{1}:%L] - %m%n"/>
+    </layout>
+  </appender>
+  <root>
+    <priority value ="info" />
+    <appender-ref ref="fileAppender" />
+  </root>
+</log4j:configuration>
\ No newline at end of file
diff --git a/samza-tools/config/samza-sql-console-log4j.xml b/samza-tools/config/samza-sql-console-log4j.xml
new file mode 100644
index 0000000000..6b0038146c
--- /dev/null
+++ b/samza-tools/config/samza-sql-console-log4j.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+-->
+<!DOCTYPE log4j:configuration PUBLIC "-//APACHE//DTD LOG4J 1.2//EN" "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="fileAppender" class="org.apache.log4j.FileAppender">
+    <param name="File"   value="./samza-sql-console.log" />
+    <param name="Append" value="false" />
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5p [%c{1}:%L] - %m%n"/>
+    </layout>
+  </appender>
+  <root>
+    <priority value ="info" />
+    <appender-ref ref="fileAppender" />
+  </root>
+</log4j:configuration>
\ No newline at end of file
diff --git a/samza-tools/scripts/eh-consumer.sh b/samza-tools/scripts/eh-consumer.sh
new file mode 100755
index 0000000000..363e028e44
--- /dev/null
+++ b/samza-tools/scripts/eh-consumer.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ `uname` == 'Linux' ];
+then
+  base_dir=$(readlink -f $(dirname $0))
+else
+  base_dir=$(realpath $(dirname $0))
+fi
+
+if [ "x$LOG4J_OPTS" = "x" ]; then
+    export LOG4J_OPTS="-Dlog4j.configuration=file://$base_dir/../config/eh-consumer-log4j.xml"
+fi
+
+if [ "x$HEAP_OPTS" = "x" ]; then
+    export HEAP_OPTS="-Xmx1G -Xms1G"
+fi
+
+exec $base_dir/run-class.sh org.apache.samza.tools.EventHubConsoleConsumer "$@"
diff --git a/samza-tools/scripts/generate-kafka-events.sh b/samza-tools/scripts/generate-kafka-events.sh
new file mode 100755
index 0000000000..858ade646e
--- /dev/null
+++ b/samza-tools/scripts/generate-kafka-events.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ `uname` == 'Linux' ];
+then
+  base_dir=$(readlink -f $(dirname $0))
+else
+  base_dir=$(realpath $(dirname $0))
+fi
+
+if [ "x$LOG4J_OPTS" = "x" ]; then
+    export LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/generate-kafka-events-log4j.xml"
+fi
+
+if [ "x$HEAP_OPTS" = "x" ]; then
+    export HEAP_OPTS="-Xmx1G -Xms1G"
+fi
+
+exec $base_dir/run-class.sh org.apache.samza.tools.GenerateKafkaEvents "$@"
diff --git a/samza-tools/scripts/samza-sql-console.sh b/samza-tools/scripts/samza-sql-console.sh
new file mode 100755
index 0000000000..39d6930966
--- /dev/null
+++ b/samza-tools/scripts/samza-sql-console.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ `uname` == 'Linux' ];
+then
+  base_dir=$(readlink -f $(dirname $0))
+else
+  base_dir=$(realpath $(dirname $0))
+fi
+
+if [ "x$LOG4J_OPTS" = "x" ]; then
+    export LOG4J_OPTS="-Dlog4j.configuration=file://$base_dir/../config/samza-sql-console-log4j.xml"
+fi
+
+if [ "x$HEAP_OPTS" = "x" ]; then
+    export HEAP_OPTS="-Xmx1G -Xms1G"
+fi
+
+exec $base_dir/run-class.sh org.apache.samza.tools.SamzaSqlConsole "$@"
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/CommandLineHelper.java b/samza-tools/src/main/java/org/apache/samza/tools/CommandLineHelper.java
new file mode 100644
index 0000000000..3cabce374f
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/CommandLineHelper.java
@@ -0,0 +1,42 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.lang3.StringUtils;
+
+
+/**
+ * Simple command line helper util.
+ */
+public class CommandLineHelper {
+  public static Option createOption(String shortOpt, String longOpt, String argName, boolean required,
+      String description) {
+    OptionBuilder optionBuilder = OptionBuilder.withLongOpt(longOpt).withDescription(description).isRequired(required);
+
+    if (!StringUtils.isEmpty(argName)) {
+
+      optionBuilder = optionBuilder.withArgName(argName).hasArg();
+    }
+
+    return optionBuilder.create(shortOpt);
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/ConsoleLoggingSystemFactory.java b/samza-tools/src/main/java/org/apache/samza/tools/ConsoleLoggingSystemFactory.java
new file mode 100644
index 0000000000..87abc76918
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/ConsoleLoggingSystemFactory.java
@@ -0,0 +1,126 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.samza.Partition;
+import org.apache.samza.config.Config;
+import org.apache.samza.metrics.MetricsRegistry;
+import org.apache.samza.system.OutgoingMessageEnvelope;
+import org.apache.samza.system.SystemAdmin;
+import org.apache.samza.system.SystemConsumer;
+import org.apache.samza.system.SystemFactory;
+import org.apache.samza.system.SystemProducer;
+import org.apache.samza.system.SystemStreamMetadata;
+import org.apache.samza.system.SystemStreamPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Console logging System factory that just writes the messages to the console output.
+ * This system factory is useful when the user wants to print the output of the stream processing to console.
+ */
+public class ConsoleLoggingSystemFactory implements SystemFactory {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ConsoleLoggingSystemFactory.class);
+
+  @Override
+  public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public SystemProducer getProducer(String systemName, Config config, MetricsRegistry registry) {
+    return new LoggingSystemProducer();
+  }
+
+  @Override
+  public SystemAdmin getAdmin(String systemName, Config config) {
+    return new SimpleSystemAdmin(config);
+  }
+
+  private class LoggingSystemProducer implements SystemProducer {
+    @Override
+    public void start() {
+    }
+
+    @Override
+    public void stop() {
+    }
+
+    @Override
+    public void register(String source) {
+      LOG.info("Registering source" + source);
+    }
+
+    @Override
+    public void send(String source, OutgoingMessageEnvelope envelope) {
+      String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(),
+          new String((byte[]) envelope.getMessage()));
+      LOG.info(msg);
+
+      if (envelope.getKey() != null) {
+        System.out.println(String.format("Key:%s Value:%s", envelope.getKey(),
+            new String((byte[]) envelope.getMessage())));
+      } else {
+        System.out.println(new String((byte[]) envelope.getMessage()));
+      }
+    }
+
+    @Override
+    public void flush(String source) {
+    }
+  }
+
+  public static class SimpleSystemAdmin implements SystemAdmin {
+
+    public SimpleSystemAdmin(Config config) {
+    }
+
+    @Override
+    public Map<SystemStreamPartition, String> getOffsetsAfter(Map<SystemStreamPartition, String> offsets) {
+      return offsets.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, null));
+    }
+
+    @Override
+    public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
+      return streamNames.stream()
+          .collect(Collectors.toMap(Function.identity(), streamName -> new SystemStreamMetadata(streamName,
+              Collections.singletonMap(new Partition(0),
+                  new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null)))));
+    }
+
+    @Override
+    public Integer offsetComparator(String offset1, String offset2) {
+      if (offset1 == null) {
+        return offset2 == null ? 0 : -1;
+      } else if (offset2 == null) {
+        return 1;
+      }
+      return offset1.compareTo(offset2);
+    }
+  }
+}
\ No newline at end of file
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/EventHubConsoleConsumer.java b/samza-tools/src/main/java/org/apache/samza/tools/EventHubConsoleConsumer.java
new file mode 100644
index 0000000000..096e12d490
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/EventHubConsoleConsumer.java
@@ -0,0 +1,120 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools;
+
+import com.microsoft.azure.eventhubs.EventData;
+import com.microsoft.azure.eventhubs.EventHubClient;
+import com.microsoft.azure.eventhubs.EventHubRuntimeInformation;
+import com.microsoft.azure.eventhubs.PartitionReceiver;
+import com.microsoft.azure.servicebus.ConnectionStringBuilder;
+import com.microsoft.azure.servicebus.ServiceBusException;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+
+/**
+ * Tool to read events from Microsoft Azure event hubs.
+ */
+public class EventHubConsoleConsumer {
+
+  private static final String OPT_SHORT_EVENTHUB_NAME = "e";
+  private static final String OPT_LONG_EVENTHUB_NAME = "ehname";
+  private static final String OPT_ARG_EVENTHUB_NAME = "EVENTHUB_NAME";
+  private static final String OPT_DESC_EVENTHUB_NAME = "Name of the event hub.";
+
+  private static final String OPT_SHORT_NAMESPACE = "n";
+  private static final String OPT_LONG_NAMESPACE = "namespace";
+  private static final String OPT_ARG_NAMESPACE = "EVENTHUB_NAMESPACE";
+  private static final String OPT_DESC_NAMESPACE = "Namespace of the event hub.";
+
+  private static final String OPT_SHORT_KEY_NAME = "k";
+  private static final String OPT_LONG_KEY_NAME = "key";
+  private static final String OPT_ARG_KEY_NAME = "KEY_NAME";
+  private static final String OPT_DESC_KEY_NAME = "Name of the key.";
+
+  private static final String OPT_SHORT_TOKEN = "t";
+  private static final String OPT_LONG_TOKEN = "token";
+  private static final String OPT_ARG_TOKEN = "TOKEN";
+  private static final String OPT_DESC_TOKEN = "Token corresponding to the key.";
+
+  public static void main(String[] args)
+      throws ServiceBusException, IOException, ExecutionException, InterruptedException {
+    Options options = new Options();
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_EVENTHUB_NAME, OPT_LONG_EVENTHUB_NAME, OPT_ARG_EVENTHUB_NAME, true,
+        OPT_DESC_EVENTHUB_NAME));
+
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_NAMESPACE, OPT_LONG_NAMESPACE, OPT_ARG_NAMESPACE, true, OPT_DESC_NAMESPACE));
+
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_KEY_NAME, OPT_LONG_KEY_NAME, OPT_ARG_KEY_NAME, true, OPT_DESC_KEY_NAME));
+
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_TOKEN, OPT_LONG_TOKEN, OPT_ARG_TOKEN, true, OPT_DESC_TOKEN));
+
+    CommandLineParser parser = new BasicParser();
+    CommandLine cmd;
+    try {
+      cmd = parser.parse(options, args);
+    } catch (Exception e) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp(String.format("Error: %s%neh-console-consumer.sh", e.getMessage()), options);
+      return;
+    }
+
+    String ehName = cmd.getOptionValue(OPT_SHORT_EVENTHUB_NAME);
+    String namespace = cmd.getOptionValue(OPT_SHORT_NAMESPACE);
+    String keyName = cmd.getOptionValue(OPT_SHORT_KEY_NAME);
+    String token = cmd.getOptionValue(OPT_SHORT_TOKEN);
+
+    consumeEvents(ehName, namespace, keyName, token);
+  }
+
+  private static void consumeEvents(String ehName, String namespace, String keyName, String token)
+      throws ServiceBusException, IOException, ExecutionException, InterruptedException {
+    ConnectionStringBuilder connStr = new ConnectionStringBuilder(namespace, ehName, keyName, token);
+
+    EventHubClient client = EventHubClient.createFromConnectionStringSync(connStr.toString());
+
+    EventHubRuntimeInformation runTimeInfo = client.getRuntimeInformation().get();
+    int numPartitions = runTimeInfo.getPartitionCount();
+    for (int partition = 0; partition < numPartitions; partition++) {
+      PartitionReceiver receiver =
+          client.createReceiverSync(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, String.valueOf(partition),
+              PartitionReceiver.START_OF_STREAM);
+      receiver.receive(10).handle((records, throwable) -> handleComplete(receiver, records, throwable));
+    }
+  }
+
+  private static Object handleComplete(PartitionReceiver receiver, Iterable<EventData> records, Throwable throwable) {
+    for (EventData record : records) {
+      System.out.println(
+          String.format("Partition %s, Event %s", receiver.getPartitionId(), new String(record.getBytes())));
+    }
+
+    receiver.receive(10).handle((r, t) -> handleComplete(receiver, r, t));
+    return null;
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/GenerateKafkaEvents.java b/samza-tools/src/main/java/org/apache/samza/tools/GenerateKafkaEvents.java
new file mode 100644
index 0000000000..6c30eee876
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/GenerateKafkaEvents.java
@@ -0,0 +1,205 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools;
+
+import org.apache.samza.tools.schemas.PageViewEvent;
+import org.apache.samza.tools.schemas.ProfileChangeEvent;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Properties;
+import java.util.function.Function;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GenerateKafkaEvents {
+
+  private static final String OPT_SHORT_TOPIC_NAME = "t";
+  private static final String OPT_LONG_TOPIC_NAME = "topic";
+  private static final String OPT_ARG_TOPIC_NAME = "TOPIC_NAME";
+  private static final String OPT_DESC_TOPIC_NAME = "Name of the topic to write events to.";
+
+  private static final String OPT_SHORT_BROKER = "b";
+  private static final String OPT_LONG_BROKER = "broker";
+  private static final String OPT_ARG_BROKER = "BROKER";
+  private static final String OPT_DESC_BROKER = "Kafka broker endpoint.";
+  private static final String DEFAULT_BROKER = "localhost:9092";
+
+  private static final String OPT_SHORT_NUM_EVENTS = "n";
+  private static final String OPT_LONG_NUM_EVENTS = "numEvents";
+  private static final String OPT_ARG_NUM_EVENTS = "NUM_EVENTS";
+  private static final String OPT_DESC_NUM_EVENTS = "Number of events to be produced.";
+
+  private static final String OPT_SHORT_EVENT_TYPE = "e";
+  private static final String OPT_LONG_EVENT_TYPE = "eventtype";
+  private static final String OPT_ARG_EVENT_TYPE = "EVENT_TYPE";
+  private static final String OPT_DESC_EVENT_TYPE =
+      "Type of the event (PageView|ProfileChange) Default(ProfileChange).";
+
+  private static final Logger LOG = LoggerFactory.getLogger(GenerateKafkaEvents.class);
+  private static RandomValueGenerator randValueGenerator;
+
+  private static String[] companies =
+      new String[]{"Microsoft", "LinkedIn", "Google", "Facebook", "Amazon", "Apple", "Twitter", "Snap"};
+
+  private static final String PAGEVIEW_EVENTTYPE = "pageview";
+
+  public static void main(String[] args) throws UnsupportedEncodingException, InterruptedException {
+    randValueGenerator = new RandomValueGenerator(System.currentTimeMillis());
+    Options options = new Options();
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_TOPIC_NAME, OPT_LONG_TOPIC_NAME, OPT_ARG_TOPIC_NAME, true, OPT_DESC_TOPIC_NAME));
+
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_BROKER, OPT_LONG_BROKER, OPT_ARG_BROKER, false, OPT_DESC_BROKER));
+
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_NUM_EVENTS, OPT_LONG_NUM_EVENTS, OPT_ARG_NUM_EVENTS, false, OPT_DESC_NUM_EVENTS));
+
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_EVENT_TYPE, OPT_LONG_EVENT_TYPE, OPT_ARG_EVENT_TYPE, false, OPT_DESC_EVENT_TYPE));
+
+    CommandLineParser parser = new BasicParser();
+    CommandLine cmd;
+    try {
+      cmd = parser.parse(options, args);
+    } catch (Exception e) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp(String.format("Error: %s%ngenerate-events.sh", e.getMessage()), options);
+      return;
+    }
+
+    String topicName = cmd.getOptionValue(OPT_SHORT_TOPIC_NAME);
+    String broker = cmd.getOptionValue(OPT_SHORT_BROKER, DEFAULT_BROKER);
+    long numEvents = Long.parseLong(cmd.getOptionValue(OPT_SHORT_NUM_EVENTS, String.valueOf(Long.MAX_VALUE)));
+    String eventType = cmd.getOptionValue(OPT_SHORT_EVENT_TYPE);
+    generateEvents(broker, topicName, eventType, numEvents);
+  }
+
+  private static void generateEvents(String brokers, String topicName, String eventType, long numEvents)
+      throws UnsupportedEncodingException, InterruptedException {
+    Properties props = new Properties();
+    props.put("bootstrap.servers", brokers);
+    props.put("retries", 100);
+    props.put("batch.size", 16384);
+    props.put("key.serializer", ByteArraySerializer.class.getCanonicalName());
+    props.put("value.serializer", ByteArraySerializer.class.getCanonicalName());
+
+    Function<Integer, Pair<String, byte[]>> eventGenerator;
+    if (eventType.toLowerCase().contains(PAGEVIEW_EVENTTYPE)) {
+      eventGenerator = GenerateKafkaEvents::generatePageViewEvent;
+    } else {
+      eventGenerator = GenerateKafkaEvents::generateProfileChangeEvent;
+    }
+
+    boolean doSleep = false;
+    // sleep only when the events have to be produced continuously.
+    if (numEvents == Long.MAX_VALUE) {
+      doSleep = true;
+    }
+
+    try (Producer<byte[], byte[]> producer = new KafkaProducer<>(props)) {
+      for (int index = 0; index < numEvents; index++) {
+        final int finalIndex = 0;
+        Pair<String, byte[]> record = eventGenerator.apply(index);
+        producer.send(new ProducerRecord<>(topicName, record.getLeft().getBytes("UTF-8"), record.getRight()),
+            (metadata, exception) -> {
+              if (exception == null) {
+                LOG.info("send completed for event {} at offset {}", finalIndex, metadata.offset());
+              } else {
+                throw new RuntimeException("Failed to send message.", exception);
+              }
+            });
+        System.out.println(String.format("Published event %d to topic %s", index, topicName));
+        if (doSleep) {
+          Thread.sleep(1000);
+        }
+      }
+
+      producer.flush();
+    }
+  }
+
+  private static Pair<String, byte[]> generateProfileChangeEvent(Integer index) {
+    ProfileChangeEvent event = new ProfileChangeEvent();
+    String name = randValueGenerator.getNextString(10, 20);
+    event.Name = name;
+    event.NewCompany = companies[randValueGenerator.getNextInt(0, companies.length - 1)];
+    event.OldCompany = companies[randValueGenerator.getNextInt(0, companies.length - 1)];
+    event.ProfileChangeTimestamp = System.currentTimeMillis();
+    byte[] value;
+    try {
+      value = encodeAvroSpecificRecord(ProfileChangeEvent.class, event);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+
+    return new ImmutablePair<>(name, value);
+  }
+
+  /**
+   * Encode an Avro record into byte array
+   *
+   * @param clazz The class type of the Avro record
+   * @param record the instance of the avro record
+   * @param <T> The type of the avro record.
+   * @return encoded bytes
+   * @throws java.io.IOException
+   */
+  public static <T> byte[] encodeAvroSpecificRecord(Class<T> clazz, T record) throws IOException {
+    DatumWriter<T> msgDatumWriter = new SpecificDatumWriter<>(clazz);
+    ByteArrayOutputStream os = new ByteArrayOutputStream();
+    Encoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+    msgDatumWriter.write(record, encoder);
+    encoder.flush();
+    return os.toByteArray();
+  }
+
+  private static Pair<String, byte[]> generatePageViewEvent(int index) {
+    PageViewEvent event = new PageViewEvent();
+    String name = randValueGenerator.getNextString(10, 20);
+    event.id = randValueGenerator.getNextInt();
+    event.Name = name;
+    event.ViewerName = randValueGenerator.getNextString(10, 20);
+    event.ProfileViewTimestamp = System.currentTimeMillis();
+    byte[] value;
+    try {
+      value = encodeAvroSpecificRecord(PageViewEvent.class, event);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return new ImmutablePair<>(name, value);
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/RandomValueGenerator.java b/samza-tools/src/main/java/org/apache/samza/tools/RandomValueGenerator.java
new file mode 100644
index 0000000000..18e0316559
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/RandomValueGenerator.java
@@ -0,0 +1,87 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools;
+
+import java.util.Random;
+
+
+/**
+ * Simple utility to generate random values.
+ */
+public class RandomValueGenerator {
+
+  private String validChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  private Random rand;
+
+  // to help reproducibility of failed tests, seed is always required
+  public RandomValueGenerator(long seed) {
+    rand = new Random(seed);
+  }
+
+  public int getNextInt() {
+    return rand.nextInt();
+  }
+
+  // to make it inclusive of min and max for the range, add 1 to the difference
+  public int getNextInt(int min, int max) {
+    if (max == min) {
+      return min;
+    }
+    // assert(max > min);
+
+    return (rand.nextInt(max - min + 1) + min);
+  }
+
+  public String getNextString(int min, int max) {
+    int length = getNextInt(min, max);
+
+    StringBuilder strbld = new StringBuilder();
+    for (int i = 0; i < length; i++) {
+      char ch = validChars.charAt(rand.nextInt(validChars.length()));
+      strbld.append(ch);
+    }
+
+    return strbld.toString();
+  }
+
+  public double getNextDouble() {
+    return rand.nextDouble();
+  }
+
+  public float getNextFloat() {
+    return rand.nextFloat();
+  }
+
+  public long getNextLong() {
+    long randomLong = rand.nextLong();
+
+    return randomLong == Long.MIN_VALUE ? 0 : Math.abs(randomLong);
+  }
+
+  public boolean getNextBoolean() {
+    return rand.nextBoolean();
+  }
+
+  public byte[] getNextBytes(int maxBytesLength) {
+    byte[] bytes = new byte[this.getNextInt(0, maxBytesLength)];
+    rand.nextBytes(bytes);
+    return bytes;
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java b/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java
new file mode 100644
index 0000000000..9803117144
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java
@@ -0,0 +1,188 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools;
+
+import com.google.common.base.Joiner;
+import org.apache.samza.tools.avro.AvroSchemaGenRelConverterFactory;
+import org.apache.samza.tools.avro.AvroSerDeFactory;
+import org.apache.samza.tools.json.JsonRelConverterFactory;
+import org.apache.samza.tools.schemas.PageViewEvent;
+import org.apache.samza.tools.schemas.ProfileChangeEvent;
+import org.apache.samza.tools.udf.RegexMatchUdf;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.samza.config.JobConfig;
+import org.apache.samza.config.JobCoordinatorConfig;
+import org.apache.samza.config.MapConfig;
+import org.apache.samza.config.TaskConfig;
+import org.apache.samza.container.grouper.task.SingleContainerGrouperFactory;
+import org.apache.samza.serializers.StringSerdeFactory;
+import org.apache.samza.sql.avro.ConfigBasedAvroRelSchemaProviderFactory;
+import org.apache.samza.sql.fn.FlattenUdf;
+import org.apache.samza.sql.impl.ConfigBasedSourceResolverFactory;
+import org.apache.samza.sql.impl.ConfigBasedUdfResolver;
+import org.apache.samza.sql.interfaces.SqlSystemStreamConfig;
+import org.apache.samza.sql.runner.SamzaSqlApplicationConfig;
+import org.apache.samza.sql.runner.SamzaSqlApplicationRunner;
+import org.apache.samza.sql.testutil.JsonUtil;
+import org.apache.samza.sql.testutil.SqlFileParser;
+import org.apache.samza.standalone.PassthroughJobCoordinatorFactory;
+import org.apache.samza.system.kafka.KafkaSystemFactory;
+
+
+public class SamzaSqlConsole {
+
+  private static final String OPT_SHORT_SQL_FILE = "f";
+  private static final String OPT_LONG_SQL_FILE = "file";
+  private static final String OPT_ARG_SQL_FILE = "SQL_FILE";
+  private static final String OPT_DESC_SQL_FILE = "Path to the SQL file to execute.";
+
+  private static final String OPT_SHORT_SQL_STMT = "s";
+  private static final String OPT_LONG_SQL_STMT = "sql";
+  private static final String OPT_ARG_SQL_STMT = "SQL_STMT";
+  private static final String OPT_DESC_SQL_STMT = "SQL statement to execute.";
+
+  private static final String SAMZA_SYSTEM_KAFKA = "kafka";
+  private static final String SAMZA_SYSTEM_LOG = "log";
+
+  public static void main(String[] args) {
+    Options options = new Options();
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_SQL_FILE, OPT_LONG_SQL_FILE, OPT_ARG_SQL_FILE, false, OPT_DESC_SQL_FILE));
+    options.addOption(
+        CommandLineHelper.createOption(OPT_SHORT_SQL_STMT, OPT_LONG_SQL_STMT, OPT_ARG_SQL_STMT, false, OPT_DESC_SQL_STMT));
+
+    CommandLineParser parser = new BasicParser();
+    CommandLine cmd;
+    try {
+      cmd = parser.parse(options, args);
+      if (!cmd.hasOption(OPT_SHORT_SQL_STMT) && !cmd.hasOption(OPT_SHORT_SQL_FILE)) {
+        throw new Exception(
+            String.format("One of the (%s or %s) options needs to be set", OPT_SHORT_SQL_FILE, OPT_SHORT_SQL_STMT));
+      }
+    } catch (Exception e) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp(String.format("Error: %s%nsamza-sql-console.sh", e.getMessage()), options);
+      return;
+    }
+
+    List<String> sqlStmts;
+
+    if (cmd.hasOption(OPT_SHORT_SQL_FILE)) {
+      String sqlFile = cmd.getOptionValue(OPT_SHORT_SQL_FILE);
+      sqlStmts = SqlFileParser.parseSqlFile(sqlFile);
+    } else {
+      String sql = cmd.getOptionValue(OPT_SHORT_SQL_STMT);
+      System.out.println("Executing sql " + sql);
+      sqlStmts = Collections.singletonList(sql);
+    }
+
+    executeSql(sqlStmts);
+  }
+
+  public static void executeSql(List<String> sqlStmts) {
+    Map<String, String> staticConfigs = fetchSamzaSqlConfig();
+    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
+    SamzaSqlApplicationRunner runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs));
+    runner.runAndWaitForFinish();
+  }
+
+  public static Map<String, String> fetchSamzaSqlConfig() {
+    HashMap<String, String> staticConfigs = new HashMap<>();
+
+    staticConfigs.put(JobConfig.JOB_NAME(), "sql-job");
+    staticConfigs.put(JobConfig.PROCESSOR_ID(), "1");
+    staticConfigs.put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, PassthroughJobCoordinatorFactory.class.getName());
+    staticConfigs.put(TaskConfig.GROUPER_FACTORY(), SingleContainerGrouperFactory.class.getName());
+
+    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SOURCE_RESOLVER, "config");
+    String configSourceResolverDomain =
+        String.format(SamzaSqlApplicationConfig.CFG_FMT_SOURCE_RESOLVER_DOMAIN, "config");
+    staticConfigs.put(configSourceResolverDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
+        ConfigBasedSourceResolverFactory.class.getName());
+
+    staticConfigs.put(SamzaSqlApplicationConfig.CFG_UDF_RESOLVER, "config");
+    String configUdfResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_UDF_RESOLVER_DOMAIN, "config");
+    staticConfigs.put(configUdfResolverDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
+        ConfigBasedUdfResolver.class.getName());
+    staticConfigs.put(configUdfResolverDomain + ConfigBasedUdfResolver.CFG_UDF_CLASSES,
+        Joiner.on(",").join(RegexMatchUdf.class.getName(), FlattenUdf.class.getName()));
+
+    staticConfigs.put("serializers.registry.string.class", StringSerdeFactory.class.getName());
+    staticConfigs.put("serializers.registry.avro.class", AvroSerDeFactory.class.getName());
+    staticConfigs.put(AvroSerDeFactory.CFG_AVRO_SCHEMA, ProfileChangeEvent.SCHEMA$.toString());
+
+    String kafkaSystemConfigPrefix =
+        String.format(ConfigBasedSourceResolverFactory.CFG_FMT_SAMZA_PREFIX, SAMZA_SYSTEM_KAFKA);
+    String avroSamzaSqlConfigPrefix = configSourceResolverDomain + String.format("%s.", SAMZA_SYSTEM_KAFKA);
+    staticConfigs.put(kafkaSystemConfigPrefix + "samza.factory", KafkaSystemFactory.class.getName());
+    staticConfigs.put(kafkaSystemConfigPrefix + "samza.key.serde", "string");
+    staticConfigs.put(kafkaSystemConfigPrefix + "samza.msg.serde", "avro");
+    staticConfigs.put(kafkaSystemConfigPrefix + "consumer.zookeeper.connect", "localhost:2181");
+    staticConfigs.put(kafkaSystemConfigPrefix + "producer.bootstrap.servers", "localhost:9092");
+
+    staticConfigs.put(kafkaSystemConfigPrefix + "samza.offset.reset", "true");
+    staticConfigs.put(kafkaSystemConfigPrefix + "samza.offset.default", "oldest");
+
+    staticConfigs.put(avroSamzaSqlConfigPrefix + SqlSystemStreamConfig.CFG_SAMZA_REL_CONVERTER, "avro");
+    staticConfigs.put(avroSamzaSqlConfigPrefix + SqlSystemStreamConfig.CFG_REL_SCHEMA_PROVIDER, "config");
+
+    String logSystemConfigPrefix =
+        String.format(ConfigBasedSourceResolverFactory.CFG_FMT_SAMZA_PREFIX, SAMZA_SYSTEM_LOG);
+    String logSamzaSqlConfigPrefix = configSourceResolverDomain + String.format("%s.", SAMZA_SYSTEM_LOG);
+    staticConfigs.put(logSystemConfigPrefix + "samza.factory", ConsoleLoggingSystemFactory.class.getName());
+    staticConfigs.put(logSamzaSqlConfigPrefix + SqlSystemStreamConfig.CFG_SAMZA_REL_CONVERTER, "json");
+    staticConfigs.put(logSamzaSqlConfigPrefix + SqlSystemStreamConfig.CFG_REL_SCHEMA_PROVIDER, "config");
+
+    String avroSamzaToRelMsgConverterDomain =
+        String.format(SamzaSqlApplicationConfig.CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, "avro");
+
+    staticConfigs.put(avroSamzaToRelMsgConverterDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
+        AvroSchemaGenRelConverterFactory.class.getName());
+
+    String jsonSamzaToRelMsgConverterDomain =
+        String.format(SamzaSqlApplicationConfig.CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, "json");
+
+    staticConfigs.put(jsonSamzaToRelMsgConverterDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
+        JsonRelConverterFactory.class.getName());
+
+    String configAvroRelSchemaProviderDomain =
+        String.format(SamzaSqlApplicationConfig.CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN, "config");
+    staticConfigs.put(configAvroRelSchemaProviderDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
+        ConfigBasedAvroRelSchemaProviderFactory.class.getName());
+
+    staticConfigs.put(
+        configAvroRelSchemaProviderDomain + String.format(ConfigBasedAvroRelSchemaProviderFactory.CFG_SOURCE_SCHEMA,
+            "kafka", "PageViewStream"), PageViewEvent.SCHEMA$.toString());
+
+    staticConfigs.put(
+        configAvroRelSchemaProviderDomain + String.format(ConfigBasedAvroRelSchemaProviderFactory.CFG_SOURCE_SCHEMA,
+            "kafka", "ProfileChangeStream"), ProfileChangeEvent.SCHEMA$.toString());
+
+    return staticConfigs;
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverter.java b/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverter.java
new file mode 100644
index 0000000000..198b84bbab
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverter.java
@@ -0,0 +1,94 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools.avro;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.samza.config.Config;
+import org.apache.samza.operators.KV;
+import org.apache.samza.sql.avro.AvroRelConverter;
+import org.apache.samza.sql.avro.AvroRelSchemaProvider;
+import org.apache.samza.sql.data.SamzaSqlRelMessage;
+import org.apache.samza.system.SystemStream;
+
+
+/**
+ * Special form for AvroRelConverter that generates the avro schema on the output based on the
+ * fields in {@link SamzaSqlRelMessage} and uses the schema to serialize the output.
+ * This is useful to test out the SQL quickly when the destination system supports Avro serialized data,
+ * without having to manually author the avro schemas for various SQL queries.
+ */
+public class AvroSchemaGenRelConverter extends AvroRelConverter {
+
+  private final String streamName;
+  private Map<String, Schema> schemas = new HashMap<>();
+
+  public AvroSchemaGenRelConverter(SystemStream systemStream, AvroRelSchemaProvider schemaProvider, Config config) {
+    super(systemStream, schemaProvider, config);
+    streamName = systemStream.getStream();
+  }
+
+  @Override
+  public KV<Object, Object> convertToSamzaMessage(SamzaSqlRelMessage relMessage) {
+    Schema schema = computeSchema(streamName, relMessage);
+    return convertToSamzaMessage(relMessage, schema);
+  }
+
+  private Schema computeSchema(String streamName, SamzaSqlRelMessage relMessage) {
+    List<Schema.Field> keyFields = new ArrayList<>();
+    List<String> fieldNames = relMessage.getFieldNames();
+    List<Object> values = relMessage.getFieldValues();
+
+    for (int index = 0; index < fieldNames.size(); index++) {
+      if (fieldNames.get(index).equals(SamzaSqlRelMessage.KEY_NAME) || values.get(index) == null) {
+        continue;
+      }
+
+      Object value = values.get(index);
+      Schema avroType;
+      if (value instanceof GenericData.Record) {
+        avroType = ((GenericData.Record) value).getSchema();
+      } else {
+        avroType = ReflectData.get().getSchema(value.getClass());
+      }
+      keyFields.add(new Schema.Field(fieldNames.get(index), avroType, "", null));
+    }
+
+    Schema ks = Schema.createRecord(streamName, "", streamName + "_namespace", false);
+    ks.setFields(keyFields);
+    String schemaStr = ks.toString();
+    Schema schema;
+    // See whether we have a schema object corresponding to the schemaValue and reuse it.
+    // CachedSchemaRegistryClient doesn't like if we recreate schema objects.
+    if (schemas.containsKey(schemaStr)) {
+      schema = schemas.get(schemaStr);
+    } else {
+      schema = Schema.parse(schemaStr);
+      schemas.put(schemaStr, schema);
+    }
+
+    return schema;
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverterFactory.java b/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverterFactory.java
new file mode 100644
index 0000000000..cf8c5687b8
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSchemaGenRelConverterFactory.java
@@ -0,0 +1,43 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools.avro;
+
+import java.util.HashMap;
+import org.apache.samza.config.Config;
+import org.apache.samza.sql.avro.AvroRelSchemaProvider;
+import org.apache.samza.sql.interfaces.RelSchemaProvider;
+import org.apache.samza.sql.interfaces.SamzaRelConverter;
+import org.apache.samza.sql.interfaces.SamzaRelConverterFactory;
+import org.apache.samza.system.SystemStream;
+
+
+/**
+ * Factory for {@link AvroSchemaGenRelConverter}
+ */
+public class AvroSchemaGenRelConverterFactory implements SamzaRelConverterFactory {
+
+  private final HashMap<SystemStream, SamzaRelConverter> relConverters = new HashMap<>();
+
+  @Override
+  public SamzaRelConverter create(SystemStream systemStream, RelSchemaProvider relSchemaProvider, Config config) {
+    return relConverters.computeIfAbsent(systemStream,
+        ss -> new AvroSchemaGenRelConverter(ss, (AvroRelSchemaProvider) relSchemaProvider, config));
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSerDeFactory.java b/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSerDeFactory.java
new file mode 100644
index 0000000000..a0523060ac
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/avro/AvroSerDeFactory.java
@@ -0,0 +1,96 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools.avro;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.avro.io.BinaryDecoder;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
+import org.apache.samza.serializers.Serde;
+import org.apache.samza.serializers.SerdeFactory;
+
+
+/**
+ * Avro SerDe that can be used to serialize or deserialize the avro {@link GenericRecord}.
+ */
+public class AvroSerDeFactory implements SerdeFactory {
+
+  public static String CFG_AVRO_SCHEMA = "serializers.avro.schema";
+
+  @Override
+  public Serde getSerde(String name, Config config) {
+    return new AvroSerDe(config);
+  }
+
+  private class AvroSerDe implements Serde {
+    private final Schema schema;
+
+    public AvroSerDe(Config config) {
+      schema = Schema.parse(config.get(CFG_AVRO_SCHEMA));
+    }
+
+    @Override
+    public Object fromBytes(byte[] bytes) {
+      GenericRecord record;
+      try {
+        record = genericRecordFromBytes(bytes, schema);
+      } catch (IOException e) {
+        throw new SamzaException("Unable to deserialize the record", e);
+      }
+      return record;
+    }
+
+    @Override
+    public byte[] toBytes(Object o) {
+      GenericRecord record = (GenericRecord) o;
+      try {
+        return encodeAvroGenericRecord(schema, record);
+      } catch (IOException e) {
+        throw new SamzaException("Unable to serialize the record", e);
+      }
+    }
+  }
+
+  private byte[] encodeAvroGenericRecord(Schema schema, GenericRecord record) throws IOException {
+    DatumWriter<IndexedRecord> msgDatumWriter = new GenericDatumWriter<>(schema);
+    ByteArrayOutputStream os = new ByteArrayOutputStream();
+    Encoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+    msgDatumWriter.write(record, encoder);
+    encoder.flush();
+    return os.toByteArray();
+  }
+
+  private static <T> T genericRecordFromBytes(byte[] bytes, Schema schema) throws IOException {
+    BinaryDecoder binDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
+    GenericDatumReader<T> reader = new GenericDatumReader<>(schema);
+    return reader.read(null, binDecoder);
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/json/JsonRelConverterFactory.java b/samza-tools/src/main/java/org/apache/samza/tools/json/JsonRelConverterFactory.java
new file mode 100644
index 0000000000..de8dec8e06
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/json/JsonRelConverterFactory.java
@@ -0,0 +1,93 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools.json;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
+import org.apache.samza.operators.KV;
+import org.apache.samza.sql.data.SamzaSqlRelMessage;
+import org.apache.samza.sql.interfaces.RelSchemaProvider;
+import org.apache.samza.sql.interfaces.SamzaRelConverter;
+import org.apache.samza.sql.interfaces.SamzaRelConverterFactory;
+import org.apache.samza.system.SystemStream;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ObjectNode;
+
+
+/**
+ * SamzaRelConverter that can convert {@link SamzaSqlRelMessage} to json string byte array.
+ */
+public class JsonRelConverterFactory implements SamzaRelConverterFactory {
+
+  ObjectMapper mapper = new ObjectMapper();
+
+  @Override
+  public SamzaRelConverter create(SystemStream systemStream, RelSchemaProvider relSchemaProvider, Config config) {
+    return new JsonRelConverter();
+  }
+
+  public class JsonRelConverter implements SamzaRelConverter {
+
+    @Override
+    public SamzaSqlRelMessage convertToRelMessage(KV<Object, Object> kv) {
+      throw new NotImplementedException();
+    }
+
+    @Override
+    public KV<Object, Object> convertToSamzaMessage(SamzaSqlRelMessage relMessage) {
+
+      String jsonValue;
+      ObjectNode node = mapper.createObjectNode();
+
+      List<String> fieldNames = relMessage.getFieldNames();
+      List<Object> values = relMessage.getFieldValues();
+
+      for (int index = 0; index < fieldNames.size(); index++) {
+        Object value = values.get(index);
+        if (value == null) {
+          continue;
+        }
+
+        // TODO limited support right now.
+        if (Long.class.isAssignableFrom(value.getClass())) {
+          node.put(fieldNames.get(index), (Long) value);
+        } else if (Integer.class.isAssignableFrom(value.getClass())) {
+          node.put(fieldNames.get(index), (Integer) value);
+        } else if (Double.class.isAssignableFrom(value.getClass())) {
+          node.put(fieldNames.get(index), (Double) value);
+        } else if (String.class.isAssignableFrom(value.getClass())) {
+          node.put(fieldNames.get(index), (String) value);
+        } else {
+          node.put(fieldNames.get(index), value.toString());
+        }
+      }
+      try {
+        jsonValue = mapper.writeValueAsString(node);
+      } catch (IOException e) {
+        throw new SamzaException("Error json serializing object", e);
+      }
+
+      return new KV<>(relMessage.getKey(), jsonValue.getBytes());
+    }
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.avsc b/samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.avsc
new file mode 100644
index 0000000000..dea6a122b7
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.avsc
@@ -0,0 +1,51 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+{
+    "name": "PageViewEvent",
+    "version" : 1,
+    "namespace": "com.linkedin.samza.tools.avro",
+    "type": "record",
+    "fields": [
+        {
+            "name": "id",
+            "doc": "Record id.",
+            "type": ["null", "int"],
+            "default":null
+        },
+        {
+            "name": "Name",
+            "doc": "Name of the profile.",
+            "type": ["null", "string"],
+            "default":null
+        },
+        {
+            "name": "ViewerName",
+            "doc": "Name of the person who viewed the profile.",
+            "type": ["null", "string"],
+            "default":null
+        },
+        {
+            "name": "ProfileViewTimestamp",
+            "doc": "Time at which the profile was viewed.",
+            "type": ["null", "long"],
+            "default":null
+        }
+    ]
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.java b/samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.java
new file mode 100644
index 0000000000..7cfbcb9fb7
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/schemas/PageViewEvent.java
@@ -0,0 +1,60 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.samza.tools.schemas;
+
+@SuppressWarnings("all")
+public class PageViewEvent extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
+  public static final org.apache.avro.Schema SCHEMA$ = org.apache.avro.Schema.parse("{\"type\":\"record\",\"name\":\"PageViewEvent\",\"namespace\":\"com.linkedin.samza.tools.avro\",\"fields\":[{\"name\":\"id\",\"type\":[\"null\",\"int\"],\"doc\":\"Record id.\",\"default\":null},{\"name\":\"Name\",\"type\":[\"null\",\"string\"],\"doc\":\"Name of the profile.\",\"default\":null},{\"name\":\"ViewerName\",\"type\":[\"null\",\"string\"],\"doc\":\"Name of the person who viewed the profile.\",\"default\":null},{\"name\":\"ProfileViewTimestamp\",\"type\":[\"null\",\"long\"],\"doc\":\"Time at which the profile was viewed.\",\"default\":null}]}");
+  /** Record id. */
+  public java.lang.Integer id;
+  /** Name of the profile. */
+  public java.lang.CharSequence Name;
+  /** Name of the person who viewed the profile. */
+  public java.lang.CharSequence ViewerName;
+  /** Time at which the profile was viewed. */
+  public java.lang.Long ProfileViewTimestamp;
+  public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+  // Used by DatumWriter.  Applications should not call.
+  public java.lang.Object get(int field$) {
+    switch (field$) {
+    case 0: return id;
+    case 1: return Name;
+    case 2: return ViewerName;
+    case 3: return ProfileViewTimestamp;
+    default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+    }
+  }
+  // Used by DatumReader.  Applications should not call.
+  @SuppressWarnings(value="unchecked")
+  public void put(int field$, java.lang.Object value$) {
+    switch (field$) {
+    case 0: id = (java.lang.Integer)value$; break;
+    case 1: Name = (java.lang.CharSequence)value$; break;
+    case 2: ViewerName = (java.lang.CharSequence)value$; break;
+    case 3: ProfileViewTimestamp = (java.lang.Long)value$; break;
+    default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+    }
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.avsc b/samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.avsc
new file mode 100644
index 0000000000..5c1e49d447
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.avsc
@@ -0,0 +1,51 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+{
+    "name": "ProfileChangeEvent",
+    "version" : 1,
+    "namespace": "com.linkedin.samza.tools.avro",
+    "type": "record",
+    "fields": [
+        {
+            "name": "Name",
+            "doc": "Name of the profile.",
+            "type": ["null", "string"],
+            "default":null
+        },
+        {
+            "name": "NewCompany",
+            "doc": "Name of the new company the person joined.",
+            "type": ["null", "string"],
+            "default":null
+        },
+        {
+            "name": "OldCompany",
+            "doc": "Name of the old company the person was working.",
+            "type": ["null", "string"],
+            "default":null
+        },
+        {
+            "name": "ProfileChangeTimestamp",
+            "doc": "Time at which the profile was changed.",
+            "type": ["null", "long"],
+            "default":null
+        }
+    ]
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.java b/samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.java
new file mode 100644
index 0000000000..c8e951c275
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/schemas/ProfileChangeEvent.java
@@ -0,0 +1,60 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package org.apache.samza.tools.schemas;
+
+@SuppressWarnings("all")
+public class ProfileChangeEvent extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
+  public static final org.apache.avro.Schema SCHEMA$ = org.apache.avro.Schema.parse("{\"type\":\"record\",\"name\":\"ProfileChangeEvent\",\"namespace\":\"com.linkedin.samza.tools.avro\",\"fields\":[{\"name\":\"Name\",\"type\":[\"null\",\"string\"],\"doc\":\"Name of the profile.\",\"default\":null},{\"name\":\"NewCompany\",\"type\":[\"null\",\"string\"],\"doc\":\"Name of the new company the person joined.\",\"default\":null},{\"name\":\"OldCompany\",\"type\":[\"null\",\"string\"],\"doc\":\"Name of the old company the person was working.\",\"default\":null},{\"name\":\"ProfileChangeTimestamp\",\"type\":[\"null\",\"long\"],\"doc\":\"Time at which the profile was changed.\",\"default\":null}]}");
+  /** Name of the profile. */
+  public java.lang.CharSequence Name;
+  /** Name of the new company the person joined. */
+  public java.lang.CharSequence NewCompany;
+  /** Name of the old company the person was working. */
+  public java.lang.CharSequence OldCompany;
+  /** Time at which the profile was changed. */
+  public java.lang.Long ProfileChangeTimestamp;
+  public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+  // Used by DatumWriter.  Applications should not call.
+  public java.lang.Object get(int field$) {
+    switch (field$) {
+    case 0: return Name;
+    case 1: return NewCompany;
+    case 2: return OldCompany;
+    case 3: return ProfileChangeTimestamp;
+    default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+    }
+  }
+  // Used by DatumReader.  Applications should not call.
+  @SuppressWarnings(value="unchecked")
+  public void put(int field$, java.lang.Object value$) {
+    switch (field$) {
+    case 0: Name = (java.lang.CharSequence)value$; break;
+    case 1: NewCompany = (java.lang.CharSequence)value$; break;
+    case 2: OldCompany = (java.lang.CharSequence)value$; break;
+    case 3: ProfileChangeTimestamp = (java.lang.Long)value$; break;
+    default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+    }
+  }
+}
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java b/samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java
new file mode 100644
index 0000000000..42e9f04db7
--- /dev/null
+++ b/samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java
@@ -0,0 +1,40 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.tools.udf;
+
+import java.util.regex.Pattern;
+import org.apache.samza.config.Config;
+import org.apache.samza.sql.udfs.ScalarUdf;
+
+
+/**
+ * Simple RegexMatch Udf.
+ */
+public class RegexMatchUdf implements ScalarUdf {
+  @Override
+  public void init(Config config) {
+
+  }
+
+  @Override
+  public Object execute(Object... args) {
+    return Pattern.matches((String) args[0], (String) args[1]);
+  }
+}
diff --git a/samza-tools/src/main/resources/log4j.xml b/samza-tools/src/main/resources/log4j.xml
new file mode 100644
index 0000000000..15aad1cde7
--- /dev/null
+++ b/samza-tools/src/main/resources/log4j.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  you under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+  @log4j.appenders.webapp@
+
+  @log4j.appenders.public_access@
+
+  <appender name="console" class="org.apache.log4j.ConsoleAppender">
+    <layout class="org.apache.log4j.PatternLayout">
+      <param name="ConversionPattern"
+             value="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" />
+    </layout>
+  </appender>
+
+  @log4j.loggers.spring@
+
+  @log4j.loggers.public_access@
+  <logger name="org.apache" additivity="false">
+    <level value="DEBUG"/>
+    <appender-ref ref="console"/>
+  </logger>
+
+  @log4j.loggers.root@
+  <root>
+    <priority value ="info" />
+    <appender-ref ref="console"/>
+  </root>
+
+
+</log4j:configuration>
+
diff --git a/settings.gradle b/settings.gradle
index e50e816215..06141ac14d 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -25,7 +25,8 @@ include \
   'samza-shell',
   'samza-azure',
   'samza-sql',
-  'samza-aws'
+  'samza-aws',
+  'samza-tools'
 
 def scalaModules = [
         'samza-core',

From 608e4e0b7635aa00a9f1e01647de8d978051d2cf Mon Sep 17 00:00:00 2001
From: Xinyu Liu <xinyuliu.us@gmail.com>
Date: Tue, 12 Dec 2017 12:16:39 -0800
Subject: [PATCH 10/36] SAMZA-1534: Fix the visualization in job graph with the
 new PartitionBy Op

Seems the stream and the partitionBy op has the same id. So in rendering I added the stream as the id for the node. Also resolved the run.id collision issue.

Author: xiliu <xiliu@xiliu-ld1.linkedin.biz>

Reviewers: Jagadish V <vjagadish1989@gmail.com>

Closes #385 from xinyuiscool/SAMZA-1534
---
 .../execution/JobGraphJsonGenerator.java      |  6 +-
 .../runtime/RemoteApplicationRunner.java      |  7 +-
 .../execution/TestJobGraphJsonGenerator.java  | 75 +++++++++++++++++++
 .../src/main/visualizer/js/planToDagre.js     |  6 +-
 4 files changed, 87 insertions(+), 7 deletions(-)

diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
index 2729fa3598..2be01af5c4 100644
--- a/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
+++ b/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
@@ -28,19 +28,18 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
-
 import org.apache.samza.config.ApplicationConfig;
 import org.apache.samza.operators.spec.JoinOperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec.OpCode;
 import org.apache.samza.operators.spec.OutputOperatorSpec;
 import org.apache.samza.operators.spec.OutputStreamImpl;
+import org.apache.samza.operators.spec.PartitionByOperatorSpec;
 import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec;
 import org.apache.samza.table.TableSpec;
 import org.codehaus.jackson.annotate.JsonProperty;
 import org.codehaus.jackson.map.ObjectMapper;
 
-
 /**
  * This class generates the JSON representation of the {@link JobGraph}.
  */
@@ -220,6 +219,9 @@ private Map<String, Object> operatorToMap(OperatorSpec spec) {
     if (spec instanceof OutputOperatorSpec) {
       OutputStreamImpl outputStream = ((OutputOperatorSpec) spec).getOutputStream();
       map.put("outputStreamId", outputStream.getStreamSpec().getId());
+    } else if (spec instanceof PartitionByOperatorSpec) {
+      OutputStreamImpl outputStream = ((PartitionByOperatorSpec) spec).getOutputStream();
+      map.put("outputStreamId", outputStream.getStreamSpec().getId());
     }
 
     if (spec instanceof StreamTableJoinOperatorSpec) {
diff --git a/samza-core/src/main/java/org/apache/samza/runtime/RemoteApplicationRunner.java b/samza-core/src/main/java/org/apache/samza/runtime/RemoteApplicationRunner.java
index 3e046afe11..1ead8416aa 100644
--- a/samza-core/src/main/java/org/apache/samza/runtime/RemoteApplicationRunner.java
+++ b/samza-core/src/main/java/org/apache/samza/runtime/RemoteApplicationRunner.java
@@ -33,6 +33,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.UUID;
+
 
 /**
  * This class implements the {@link ApplicationRunner} that runs the applications in a remote cluster
@@ -57,8 +59,9 @@ public void runTask() {
   @Override
   public void run(StreamApplication app) {
     try {
-      // TODO: this is a tmp solution and the run.id generation will be addressed in another JIRA
-      String runId = String.valueOf(System.currentTimeMillis());
+      // TODO: run.id needs to be set for standalone: SAMZA-1531
+      // run.id is based on current system time with the most significant bits in UUID (8 digits) to avoid collision
+      String runId = String.valueOf(System.currentTimeMillis()) + "-" + UUID.randomUUID().toString().substring(0, 8);
       LOG.info("The run id for this run is {}", runId);
 
       // 1. initialize and plan
diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestJobGraphJsonGenerator.java b/samza-core/src/test/java/org/apache/samza/execution/TestJobGraphJsonGenerator.java
index ba5c922c49..b48c82dd89 100644
--- a/samza-core/src/test/java/org/apache/samza/execution/TestJobGraphJsonGenerator.java
+++ b/samza-core/src/test/java/org/apache/samza/execution/TestJobGraphJsonGenerator.java
@@ -27,10 +27,13 @@
 import org.apache.samza.operators.OutputStream;
 import org.apache.samza.operators.StreamGraphImpl;
 import org.apache.samza.operators.functions.JoinFunction;
+import org.apache.samza.operators.windows.Windows;
 import org.apache.samza.runtime.ApplicationRunner;
 import org.apache.samza.serializers.KVSerde;
+import org.apache.samza.serializers.LongSerde;
 import org.apache.samza.serializers.NoOpSerde;
 import org.apache.samza.serializers.Serde;
+import org.apache.samza.serializers.StringSerde;
 import org.apache.samza.system.StreamSpec;
 import org.apache.samza.system.SystemAdmin;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -48,6 +51,12 @@
 
 public class TestJobGraphJsonGenerator {
 
+  public class PageViewEvent {
+    String getCountry() {
+      return "";
+    }
+  }
+
   @Test
   public void test() throws Exception {
 
@@ -147,4 +156,70 @@ public void test() throws Exception {
     assertEquals(2, nodes.sinkStreams.size());
     assertEquals(2, nodes.intermediateStreams.size());
   }
+
+  @Test
+  public void test2() throws Exception {
+    Map<String, String> configMap = new HashMap<>();
+    configMap.put(JobConfig.JOB_NAME(), "test-app");
+    configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), "test-system");
+    Config config = new MapConfig(configMap);
+
+    StreamSpec input = new StreamSpec("PageView", "hdfs:/user/dummy/PageViewEvent", "hdfs");
+    StreamSpec output = new StreamSpec("PageViewCount", "PageViewCount", "kafka");
+
+    ApplicationRunner runner = mock(ApplicationRunner.class);
+    when(runner.getStreamSpec("PageView")).thenReturn(input);
+    when(runner.getStreamSpec("PageViewCount")).thenReturn(output);
+
+    // intermediate streams used in tests
+    when(runner.getStreamSpec("test-app-1-partition_by-keyed-by-country"))
+        .thenReturn(new StreamSpec("test-app-1-partition_by-keyed-by-country", "test-app-1-partition_by-keyed-by-country", "kafka"));
+
+    // set up external partition count
+    Map<String, Integer> system1Map = new HashMap<>();
+    system1Map.put("hdfs:/user/dummy/PageViewEvent", 512);
+    Map<String, Integer> system2Map = new HashMap<>();
+    system2Map.put("PageViewCount", 16);
+
+    Map<String, SystemAdmin> systemAdmins = new HashMap<>();
+    SystemAdmin systemAdmin1 = createSystemAdmin(system1Map);
+    SystemAdmin systemAdmin2 = createSystemAdmin(system2Map);
+    systemAdmins.put("hdfs", systemAdmin1);
+    systemAdmins.put("kafka", systemAdmin2);
+    StreamManager streamManager = new StreamManager(systemAdmins);
+
+    StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
+    MessageStream<KV<String, PageViewEvent>> inputStream = streamGraph.getInputStream("PageView");
+    inputStream
+        .partitionBy(kv -> kv.getValue().getCountry(), kv -> kv.getValue(), "keyed-by-country")
+        .window(Windows.keyedTumblingWindow(kv -> kv.getValue().getCountry(),
+            Duration.ofSeconds(10L),
+            () -> 0L,
+            (m, c) -> c + 1L,
+            new StringSerde(),
+            new LongSerde()), "count-by-country")
+        .map(pane -> new KV<>(pane.getKey().getKey(), pane.getMessage()))
+        .sendTo(streamGraph.getOutputStream("PageViewCount"));
+
+    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
+    ExecutionPlan plan = planner.plan(streamGraph);
+    String json = plan.getPlanAsJson();
+    System.out.println(json);
+
+    // deserialize
+    ObjectMapper mapper = new ObjectMapper();
+    JobGraphJsonGenerator.JobGraphJson nodes = mapper.readValue(json, JobGraphJsonGenerator.JobGraphJson.class);
+    JobGraphJsonGenerator.OperatorGraphJson operatorGraphJson = nodes.jobs.get(0).operatorGraph;
+    assertEquals(2, operatorGraphJson.inputStreams.size());
+    assertEquals(4, operatorGraphJson.operators.size());
+    assertEquals(1, nodes.sourceStreams.size());
+    assertEquals(1, nodes.sinkStreams.size());
+    assertEquals(1, nodes.intermediateStreams.size());
+
+    // verify partitionBy op output to the intermdiate stream of the same id
+    assertEquals(operatorGraphJson.operators.get("test-app-1-partition_by-keyed-by-country").get("outputStreamId"),
+        "test-app-1-partition_by-keyed-by-country");
+    assertEquals(operatorGraphJson.operators.get("test-app-1-send_to-5").get("outputStreamId"),
+        "PageViewCount");
+  }
 }
diff --git a/samza-shell/src/main/visualizer/js/planToDagre.js b/samza-shell/src/main/visualizer/js/planToDagre.js
index 0421c338fe..77adaa2cfa 100644
--- a/samza-shell/src/main/visualizer/js/planToDagre.js
+++ b/samza-shell/src/main/visualizer/js/planToDagre.js
@@ -36,7 +36,7 @@ function planToDagre(data) {
       labelVal += "<li>PhysicalName: " + stream.streamSpec.physicalName + "</li>"
       labelVal += "<li>PartitionCount: " + stream.streamSpec.partitionCount + "</li>"
       labelVal += "</ul></div>"
-      g.setNode(streamId,  { label: labelVal, labelType: "html", shape: "ellipse", class: streamClasses[i] });
+      g.setNode(streamId + "-stream",  { label: labelVal, labelType: "html", shape: "ellipse", class: streamClasses[i] });
     }
   }
 
@@ -67,7 +67,7 @@ function planToDagre(data) {
     for (var k = 0; k < inputs.length; k++) {
       var input = inputs[k];
       for (var m = 0; m < input.nextOperatorIds.length; m++) {
-        g.setEdge(input.streamId, input.nextOperatorIds[m]);
+        g.setEdge(input.streamId + "-stream", input.nextOperatorIds[m]);
       }
     }
 
@@ -78,7 +78,7 @@ function planToDagre(data) {
         g.setEdge(opId, operator.nextOperatorIds[j]);
       }
       if (typeof(operator.outputStreamId) !== 'undefined') {
-        g.setEdge(opId, operator.outputStreamId);
+        g.setEdge(opId, operator.outputStreamId + "-stream");
       }
     }
   }

From cb92cf18b5b1707d98a442a32a9f6c3767e13527 Mon Sep 17 00:00:00 2001
From: Xinyu Liu <xinyuliu.us@gmail.com>
Date: Tue, 12 Dec 2017 17:07:07 -0800
Subject: [PATCH 11/36] SAMZA-1512: Documentation on the multi-stage batch
 processing

Add overview documentation to explain how partitionBy(), checkpoint and state works in batch. Also organized the existing hdfs consumer/producer docs into the same hadoop folder under documentation.

Author: xinyuiscool <xinyuliu.us@gmail.com>

Reviewers: Jake Maes <jmakes@gmail.com>

Closes #381 from xinyuiscool/SAMZA-1512
---
 .../hadoop/multi_stage_batch.png              | Bin 0 -> 65364 bytes
 .../hadoop/unified_batch_streaming.png        | Bin 0 -> 17371 bytes
 .../versioned/{hdfs => hadoop}/consumer.md    |   2 +-
 .../versioned/hadoop/overview.md              |  46 ++++++++++++++++++
 .../versioned/{hdfs => hadoop}/producer.md    |   2 -
 docs/learn/documentation/versioned/index.html |  15 +++---
 .../versioned/jobs/configuration-table.html   |   4 +-
 .../versioned/yarn/yarn-security.md           |   1 -
 8 files changed, 58 insertions(+), 12 deletions(-)
 create mode 100644 docs/img/versioned/learn/documentation/hadoop/multi_stage_batch.png
 create mode 100644 docs/img/versioned/learn/documentation/hadoop/unified_batch_streaming.png
 rename docs/learn/documentation/versioned/{hdfs => hadoop}/consumer.md (99%)
 create mode 100644 docs/learn/documentation/versioned/hadoop/overview.md
 rename docs/learn/documentation/versioned/{hdfs => hadoop}/producer.md (98%)

diff --git a/docs/img/versioned/learn/documentation/hadoop/multi_stage_batch.png b/docs/img/versioned/learn/documentation/hadoop/multi_stage_batch.png
new file mode 100644
index 0000000000000000000000000000000000000000..11425400f31a19ca5a711067defb6b8d99ca123b
GIT binary patch
literal 65364
zcmcG$WmH^2vo?whF!*4>Ed+OW4-i}vY;bpXmqCIi5Zob165QQ_4=_k@cY?daCFi{7
z+<X6iYkm9A?%mT})zw{HRsB@2iB?mQ!$c!NgM)*^l>hKv0}c+M84eEK0*L%tbE0q>
z@cM#x(~y&bs~98Oe=Y0UXz94?C@Tq>IXSSIm^+zTuz5Q;zqW#d6Y&;$Ejn1Zn^1W>
z*gLujd5gaNuNFeD<$u-eZ>j#PiMyTXTODOJDrqNI3o2eV9yX4*VrW!UR3fhCmO>iu
zW&hLt^-J`vwY$5s5Ieh<mlvBCH=C2I6+5S(pddR37dsah>uU>EHy=lL6K_^WH=6$@
z@;`LmTez9I+Bm!0I5|@NqibU7<l!#*_U%7L|NH#+INfb5|6fjyZvPSMRY3NCzp!(%
zaj^d{?bog%|7wL)U2QC0Isc<C#wqe&E&qRY|KTIT{*U?pXE6UQ>3`L}iYkUC!v4S0
zCWfZeL$3=5Cjlq_UQ){&{?Gu$U#o9MJX6Xk9~qY_KNVI~miJx~{`EOSB>^DGsN-Di
z<5Aw~u0<mu!OLV|s`;Me&%*qaKLFeD6sU#*cg0(4msnbduD?R>p2k#~HD7Ly59i&+
zv#2$SKx#k+06qd5JS133f;#X2#wR91NJi7=dlUpo5vZPQe^lj$^cSfvGz40~PE<F}
z9sHeEXonKS+u0{AARwzFHlHN(2Gn))OUzAxyTi#Vjerrb6cXH7e$)lG8uJ!9`$g@2
zkrjyLOnvNRT^bIPGv4hhp4sQ>{5Uc2PH1)3ZzLTat4~EB*jqISUN!VwP|`co2cP?y
zLB1z}>%2`lUnIB>>e@yme2y;|AN-rEC!X%qzHMk}FkWbd?MJp109&^Bh*%qgFv9sS
z!}|FAyB$AhO;Gp?hh47O+C#i91^w%Q>}9)FXFLU~NyxfyXbde&PoGVzle(jL&n2;{
zz2R7_-q2PYbCLA0nkXe@S##25@=@>^4zyWu7qYK~20C%r5o)L~4Q~VOH}5v*yKa!i
z(E{zau}|sAP$co3Lx{KAl!++&LtLD=`f-?C$>7&e<Esjzb<DD@h5vfDmU`yE@cT&T
zXI!4fkS1j7$_ZGKdnsk=Tq1~%LMl0EoWyF2Jjf&s!=Y;VQe7&5D%lO<^VE9ssW4+U
zi$-sA1}>^#Qq*|9cgndVf)W0AW~}UMMqE6g2`Eg>26Z23`B<gMHJ)Vi$SkK1Vws@W
z*a;7!h2`y-QS`)~DVfgT4!`Ht$CGf`bRmE0<IuMat3|?GR|U`7E^}0Pn>}Ze*`hj_
z`lliGm(s@4hZ0-7OW$1|cpFL_Fm2Tjt_yFowoK-xpN4nbGg^%F5&Y(fjnG8z{i$og
z@5RY%#SlxfNxTLO7^uHd5X<xqth>v?4?j`hscn0~Aw#i~PwB4xN-%V`vL>~MZr0Hi
zTP?%M?1}+@)GmYGC5V7S=f7H*zWuaWI;gk-x@zIJn^jm!+JUCb^a97^gl|_0F*ceK
zaG>7ETfV_9Utyj++`}1oe`UG#|Ds#GinB(HWV0LH@irD361`PRG%C0t783TT2@#}W
zMRn`Hz@7<A*z$1jPR1jO^o(bX^BI(T+0)*LIYUIOBkhh2#7VG})s-Jiu&kiwhDy~D
zkH%itU75G;A!_9kU%-h`MOXiZCH@o(ActRax@+5{P(1%rPrlhZSJg1$GC-sG1s)z%
zp$-}3FTdSs!lLli*BS6qV3EB@_5wECa*jG%C}KG`Pd5R}vgui4OND?XssvapWyOfZ
z`?*A;H>obT7$WUlLnNbk&bz(g^0H;f?ozBlRvSV)a|y&Tl)!xOV500|8vjnhxv@2V
z3%BCuZQ!|?IdbHV&sjfLCuy%ASphpsvYtI`?46|`KTGk#0TD|7&p|if+FjC35X-<e
zcID%G(hb^;$_Qsx!{uQCD>y0qfpJmp$pl(Mu8SPhQfGnXy0C!lL1^}*T*DQ45$E%F
zej?b^N^2zvXx8qH_2mYYyJP%ZXfjX7GCk56mn&U}OR@aj`8bDk>DTZ!@11QZ7lm1@
z>Ne<x;%-gH@~Ae%g^CsRM*m@nY`gBdxx2Q2JO-Q#^9=uDVf3mPclng*3C>@e&eyWc
z;_lKYcNh=yRM(8-8qScC4aSzf>Y`EMWO$1VSoi_yIps(%)QmR5tteL~)cfsfuohAd
z#41KkqT1pG5yaJaxX%f>UpL&Bl@YSuUwzI*B?NaU2+s`-h=06y3ZY?XXBcnG2cuqH
z--&rVFBhEDik9V_SsW6o&+XR)<7fU0V#}?79t@J6*vohCV$E~trx+sH1I2gHp&vGp
zLprQvN%%2msnrUMwv<hgl`UjRV$?7$T+Muek$GNeE>E$X@thfr-MZ;ODNkJPzw-SB
zQP0~w1L~Y!;DM$P#8ZjIHET;X7VhESR3`5(+Puf)(n+D6+zA%_J*)#9pEuLQ`-5Hg
z6#dAbUe`P702EK)g9R@_*k!;KIPFYXm`*!nMQjnlv5AeABU6;hJ-{)-)pSCY{AC>P
zzDrbWI9AA5&Cc?-%ZW=s6la|4x$uZOUi%Pdasux*QHSE7w1G@KHQZwtPXFJtex75*
z<CwO_rrg`JSd)IYP@$wWVJ>Zxeve;>Cmh&Z{pu>u&9Xim$NV{*zmPbgTt*d0H4gJ{
z&ef4^)Oc#`QO1AML-&Mpi0#m4niruA+@G7VFZj$zd~jrC$uK^f%7VpW(i+K7o*(--
z2Dl5M@61S^jFt9C&AQsyiKe=GaqBmglI4VrW8DPZur2SPh2--$H)JP+WTPwxhugPW
zJ8aOf*gF)#-y)oR$q$jlmvr2di)m(QkwjBO{0teyv)xH)6S8fbaV~*9u@CP?2K|O~
zKl_t~jQjYXcToT4z7lrhseLD6R09RY``I}Rc<XU*3WG*tIs3Sox!zJaef{oCpIT<Z
z!k;Uh+&!8K$Vc<>hJ!l=nfc;IeuZ9`lN`yDbmNF#=$OfV<%}bxM@7dQRx<5RKzf5y
zM4c8|6--5*0GOh)EHX>v_<Csd6<P%<AWu#*lOSE=hPs}$8$<1e^4~S5vGQ5|c8&7o
ztbzOQ4XT4d>#+{}>ySiDwh_5OZn?%<S;#6cDiA7EXdZzk8X@6zY(rk_PvJt@C7mXT
z1LeZDkq2R%9@-MpB_m>*$&!t?y%Vup@O<VhJ_CQ~@j|mGu!x`|>u~|c{Ml0G+F8$d
zI{<<!UIUt4d7HIlbH=SjaIzzy(&^Ia6&<ghoW$7xq6(J-KX6cEImU&5F%@{fqLSXd
z?rOHO+b;sFs9~!K<S#~{HYu7X8*X?Y+RWUN!8#+QT8vr~&?y#V0!sY^Q{o(3$fp0(
z<66F_V$Sk_eongNiHgW9=zFyW<F>|h^-o;q@A}*TcR2mA)>szJ&Sn&$Vg3GfjPf(z
zI+@dicaVF8SQaIP7ScMps5=Jo&Y%p4S+v>f!1K4SbMS-!AEdijZ!GJEFs|3eH?@2Q
z=jF9laGks6=`PDQzHV;$$o<9W(<4LJClZ9S=6Rxl&^!YX9+_XV2{si)M4LK^k~QVM
z5k<EmMbTzB-(t|-crU>m`VVl(HZs6zDkK*;hf}}HW!TRl*l8h>$~#`^<6MQJEFCyd
z+H^@hmIk64r69Ce`OHuOWya`Q&tFZ?h}@?sv!pI$7sx@q7++^G7_atG0mW|g37n~>
z$G_WxiFwk9SJ4&2cWY@cOf7*D)lj{!;}zHG;uWl2XC(bR_g}^(>o_Y5IH!?I>QD*1
zgb^NiMZwd*S!K3;XJgoh{h|d*KT~rA*v`IBPp6_G8-DPIrg_P^VXDv1%8{i8)N`eD
zbJJGZ<%&8i>0E~7QSV}yMRa*33%1sOFm&p**cRZROT5d%Y>pVeB6GLIGU<1k5wHYx
z_}!{l$XMMK{6^W{@&bl7kV!MJahUi^LRr&X$X7M*NP8!g!uh6R1#1GL#cNK8D$BY$
zL8qzp%Lk<!F&j?d?0@lh456T63C4(EG!kjR9gmA&=b$#P$7rE7x`W-~Y`=vO(Uh8{
zo0M+z9>A8(b`fTtS#Tf_(N|7?A+;kdD;u<#Mw3R1MMgkkkv<lKK_|#3J=`#^rOY{>
z^GQNVB#U_S%dD5nqqZe$_8&T}rE=iq`b;F(W{yZ-?5+<RcTT1s^mawtq+`%tW!0@;
zAIh9|_RGOotd~VzOd9g)^A-eZEYigV;;keVt;e&r1nQ2~(h*FsKDR|mWK<u}l7-fr
zKkw2>`b`iG)LOBRuAya)G9B%>+nMEfmQVMO{)jRceg=2NURiNmknB`LC!jMQ#5xa}
z;ra_`F9bLB0xYfV&K^@NWS{=X#&1*!*81lN+I}(5J%2lyw)kNuhHHw5V$&JOy;GZh
z_tDb7@@v}Y5YFT1qMFUB<iE(Wl4h4ZV~HzPEBuR{^Q-%I9|nQm(7;-uIz$pDL<naz
zekq%-IH<fkC@youM2+-O+;5IeFsiFO{_q4N0b_8_^EpE#;;Tg6I4yypI%yK3m?0n`
z^QE*aiNr3!(ibE*0!z@&Bsxwp!1A&+hWOG+nrb0%gviJg<2wgaHN()|RorY1T2V0W
zD8$^rfB%|G`huRff1NCMD7G0f(%N#Ze2^Qs=F=<MbwO@+h<iJ7qD(T7KzuQZC{mKQ
z4sD9Wf$HfwC|z!W#;H4;a!D`DsdAkzqpj9dcdRW%%)HV@QO7a+!w=S>eQA`I$U|0R
zi5j^%vc{>^(*6l9?A=YOacSE^J?g1yS=9t0=rxtr@rB}-w3VC9DF+Gu9fBiohjy*G
zjGof(h};0Rw8M7L<Z<@<uG*ODwdQ087alzF4g4J4Uxl|xQ73>>T5e~AY%V~t>RLL%
zQ1i;V)E;G!hA^ibr;&x@7{+}Bv7V^^>^qxHE>(yiB7Bhkq8svG$1F(B;@Ic!dC^FF
zvDGX+Vf_Mzn}ZP4e0#g3YAAu($~!Dc=`?lZ{CDa6Lunxp^aYOO#i7_|81`fh2Ee{(
z;#nL@7`v}305F;87>(rufi}}e2YPhV8A3H(<4a2{E{(#7r!#cpPZ$O)$c;l~h%TKh
zrw52k)79ysiG4VsdES;qh3WM?nRtV8xkSPnd+Zs+b!o&tXV5g^ijeJqwbHlPRHwo>
zArHpU;{4YMlzK~~$=y#Uy|I1~m+F>vAy(Qcl0J{Yj%G6rq?Ph+T0k6cF*QrUT9$;}
zz6I=49W$Tlq*X{6nf8_d!a0QuQD)+eeDN2DoW|%2HSC)J5$<Eb9Hgyvy<eR1ywVl)
zO~^AE81#w!@=4JG=cYV9AHpOTJVa8aTv@ywi`Jt2lLxp%54`!R<Ehzu2BX3xMbmHw
z+#tI?^xS``*dvJF&GOU<<`R!H4CiCq%jVeAgl?vkQd$+n(#HSMN<2a^61kXwmg0^7
z`fjaaoMn6&luW!`%oWzpJ>r`r5Ewyh@kJXVxImP!tu_gYUlWp&Kl{%SXPhQ@m*@tH
z8{|mV&R%|m3$xx#JA?YfB8bi;nUTx5)dMh3Hg9EjuAncL{i&4UG#jT*{K3s+-0?^T
zzaP;quq<JWJAfUar3f5%+5pF^5y=(ik;Q@D%5`KAR}M(vKnIg7oWmNp<Twbea{3-=
z+-xj$ku%E7phW2LYs%)&p?7|kFx{PR(APfsI(?5uJ7~}v%y`SUkA9JsthryDT%rU=
z%>p5JAZ}pglqL7(p2(v$T;^)p1Ba10vWY)Z8mA@93p$d<IZg|;MwDV4mJSO(Q|KXb
z-ePi=rfec~;vC=#Q=^!rNn_@$w2)m*OMN>|9emPnV=2N85A_aouIw;P8K*v;^4Vd6
zdgpxN*o-6`e|Jp&7t>P3SjB8*Bx_LQn_$=-7k7tTF2{|PEaeM#gg5e;hzutRXCWBF
z4Xu_|Z#dl=Zxuc{9V!w<Y^JqEgfIbS4I$1DGiK_X;u_V(-Xh;=Yia7jA81Rvquh~#
z@}{|{5^~-c_g^4=er1!>!-9H%I78`w@|}<3>WqcOB&G7HP%x(}wngV>RUp1qe@l8K
z@i^uTwunB}FL52OVa~$GpHQ6<HZC(mZxvEbdX7S6ky@{`Gq8zL?dv+29SQffMH9Ql
zo5}|HrEwzHL8?EQ^oxAr=<M;2n#Xn6Rf6(TaxZkxU2_&z!b6D&_lUT6kQavAl8Bq~
z44}#y2y!!=W?4j}3#0=giNvdTzZ#n&d`zBu)4e!0h5dr>aJd(j`(jVtFoY7{IPqRM
zTEcpvxWXbdpZzKSnE=`X?`sy(gYy<Ng8u?XW^t@XX@cR<B?WU`ct!=Mxux^e{s%mH
zjttgYd^*V~!(o1fQrKjvLKj=RG4ijexOo(~M58u8{|8^`dI3ePQIFJv%a5MDUZ$h|
zQ6{7gp6%95zu_9-X15AV(*5!u6y}y4!Pens)vrXh;{OzraiYq*h%;HDggpC#<tO*0
z<q35nxVU;wjMO<H5`4hDOXm=`=u4rd1HmHZNnnMGD^Fkl!39AvDNcxFi`p>kBXlCE
zZ|icOkQ-s&x%XUV{WGJshQGgm$7N_^&&fVlk4F*q)IQ?cUximV#|dW;R+~8t7htRA
z+n>!7&_ADHLHh3Y$Q&%-T;Cy#T|65Oe)^W($3Rs`u9`23UJtUQ{fV@A_wYlHgQ}~8
z_i8|AFRl=7*>>`K<phDP>&$9*^q+$;sjnI>dmjFiIXMfb{%Cg(S%>7}>|m=aU%f7=
zmE?PZm|u;J9hIr8L4iT}R6X~lKgZ25w|!)*gk#ixr>{F{e37D}r*V$#+#K+Z6E2=T
zxY@sgKj3z6!K_!D9Xspr|8qa~GY5UvD*>(fUO%&!QM8GswdVMDr~^-s^f!sjq*O+u
ztjwJ3!UJm(r0-8<%%`V}Mr5X+%oftU+iBS0SDn+Dbia*Q%zmmxd4#`W!e7}PzYz1a
z@$&Zdm9Tqb-Fhw>ztj9d{P(j6E3YRc_$R1~v_(L4UmKvd2DSZecjEf7w_`G1Z<Tcg
zwPePt5xv<rz#g>BuoT9ta#G_AUmpGz68yT3v*l|c{)Nm7YB|6y(#;x<&riGKge`u;
zt2&eWzIsi+YMcv_4Q~R&K0^Xe={c~BmNcbP^ACFT^sKisAExI1=nx>Rybp#DK9*${
zqeIg)>feiYYU)^5YaX<|!HjW<ob_saodCPVI{@aBGwqUIn+?;HHVCv}@T6w&rC(rd
zYS-G%U%}ehxZi)~nD@N7`)(yzmakNeNG|uDhrF$dUr0ig#yjzE&i!S!M%V=+U{a@F
zr>kK<_pZ=WyZppc)Woy$#1Y5!ic$YSQx-1%DYn7x_D7XeS1a(BC3gmVOHW)r^(ks-
zz^Ce>WF3mw$@6({Id`Gu)BA1o0DCi+gg3&jfk6SH-?2BvSzt>{8n2)wI664~lQ}3r
zw^l3rPlf%AfkWBVa-jK<!Kwk%7TP$_cbVZ0zF}3+-j4{WE_%?1P4(EK9)7Wz{Tzv<
zpLNV<{W^i)o?5g~)>E?&gn3KVep~Tepo|~J0)ZG5Jx&e2Tv@j4p+;hwwy?nPuDal^
zSZx30SjqB}cro>c$`dv`SKY4i?%r+fAGXr5g^kL8E@%OlF3A+r%7h!D+I)+u27Urm
zG{#Bn`5SU(g5T&@k22~NQ&rA((l+N~lx+22^g^QTeVc>3qt&Gy1PIL-A^0mr*D@=a
zkf5m8*tShOH|>K~O|RcOSK0iu!m7xN`V0N6W#p@4y=8i*sDQa=UXFm0=X<_u%QbRS
z_*5f1W>-+iX{*<H=cchis+O?7r`Jg9P3VShS3oYQ7D9!h+Q&GCSKu-)G2XrR<c-pN
zIA#vt?75oc>+VFV8$D>9l47%Sei_|CMfYxM9@8$-Z2a)?8U-*%S>}k0-Vz`pI!7MO
zPwJ98k#o!+d37UXztYB8LPL0cSr{BlyVQ7Ya}Y<GcdrFpk_)~8?l+K&b+YdSrg)jz
z26&-9^Oc`qp*K5Y_9Gm>qxkD=S^1dQ1fcs9B}HWixSgMQ8XmvT`xVoCZyPboGgy%m
zO}L28?#a|wNP2DRV<6%mc;ek|$REF3J|n}v;{T^!N+Jx7dHpg;7nU7<QcAa~s9xXE
z3g?|U;}r=3w-|ckW7njjBmc-y1KuJctnO9z`5a#HQvzZAloDrQoDHYbMA*4vV6H={
zq3xyBpE-XuuL@tS>|9uIhjueMt(i+fo@hPT;8Xbtn>5ej)iO1;c95CA__g9q%Sc2S
ziN-&wLL@`ufrP3U-yDVtL?%?@Poz)M5v{SZmdRx#z4-UPMa#zr6-@)4C_;Ap&98;t
zxJ$S5;=PU%=tDZ=)d~Rz49BPV*TAd^G1YksL8t71%N85^mbGi`H5Qib3wC3F&pOA(
zXi@*bpo=cJb)>@2;{|!h#jm5Z#wo>?B$TtU;}#`&RgsazG0%WT#X$>pw2urlW+d3-
zDE4|_<gd0LsEx4^yst}Y^7JQj6n^|_1i@sQYOQ)J9I|9f_*Po*to}4ebzuUvhCX2H
zU~$v%*Og~n0ufrG4qK{q2mka&$dnA2R!~G8s(qYw6GFJX5f>{_)fi>B1B4qGML6=M
z4u8r_^icf!<JDuW(E@C%(kJYl7WaH@j++bgGa=yNpaVDABH0QF8D$OXL{Wt7wsk$0
z2a__~90@q#x-f<(%45ay=o~okRX)g>5}|4$`!A^>;S&GQ(n5{0RK@e~7_gx?K(sNm
zXr~-h6&rkv{_M!Wnq%SY>h3rP!KIn22WOIr%JT|BB#tHX0-9rJ#fL<vMn$>n<p;Nu
zaw9n2-MM13*cPL~XM%v=^LYIE@weXs*0c;&TfE#RlK*HCAP7>zi(>XOLIi^fn(ZRy
z9jT^D?6;?+lEs&nu5?Q07h;1AZ{FCs1;HRZAF7(;nf4liT}cS){gq=XRD%0Fqpuri
zS}7zHiQ$)&;OD@<F2kApm5i%7gWpE98H&84O3xSBgVfdwAZ2b|z3Feu%F5nSI#hyj
z6;W;|3Q`}gi{66!Ke`|6u<G$xp&B&%e}op=Vs6{_e4P8?DH19H%GJ0<aeDt#vmJaD
zL$R;!vEd1*A#i0Bbp|*IxIIW{p`gKw%TPttHw(No&C889!d#REY;0~`EPqdy2a7y-
zaA)!*;U(EjM{!~&+s<85-v?a3N5eq#U(@?UyeX$#q&u|(+rYedUp*xLGA8Ag26FXs
zSe2%{eo3PnB~mrn9+~(o!`yrb{`)8;mx|iFZ?k`C8c8JNy2n6Z9`f{HB0gz!5mb1O
zd0@$F@1ab+W(N${I5n_mIzF^R)ZBEz2w@o|m^%yLL2oK2v(x*d0o=j`%o%2{dV7yn
zd$UQ-A51B+rcEZGtZQ6t1!LdiQ6ee{Z#KfrRQXDd^gUt49+}E+qP}}+gROx_5scU*
zi?t==I6Nf}o<}7yyX=#cc+a>t)=`qHPh6qT7k7~WOt!k8^qY9iW4d1g9V95JKcuo4
z^i0z&$gNr{D(;~?<&y(PqVx+qs!8|OI;1ayXr2|BH&f6YE`M`svJiaqL4#kFl~B#b
z%2}LUXhbQKwPHrlwR?9tJunlY^{QpBfgPm<;P>g{OKJ?~;DlzvE@p(+$%LAhGVTr<
z%(p_nK8MO#D$T!4(<w9&J=BsA!F-ltd}3C*^LzbIJ|OF1bm1AkUPUa(SN(V>q8tKy
zG+f0Fr}OH1aDw5_{0+^sfuwj}Up7l6wc0*t<j+5u+*;q?4H%&==)l~5tv_wFj=#@k
z_w-I*#QwfjWIln?&uP1HGhH)^SMcj`ZWMu|)U#dB6#QpS%jD*I`Ml;eat(w?1gnT=
ztS(JQ|MMx@h}vNRaPU7fz@n{#>&v}Il%@Gx|AJFR;ALHM*|{q+LfH)h%9#@QSl+B}
z<KyMoV%R<+AN}q204l(wai}nEZ?D@-swu14p2R_{E=A!Ro4lJ93r7fQ95YX+xXNQZ
z-4l0?!np@WxDM-|s2|kfpSA+V7YvH(>pXh-NG_Jgr++vo>NQwUUj<d>f`aMe+bbcE
zhTD=Maad@*Gb)rmU+GzYF+56I5raviE(&<DHDZU`CKGZ0YRCx@J{}V&b$c0p$)zzi
zNb;+-Mqe>rKbKh^Khx;d2I@#tp=I)TKr#+%K1pmp&?5+J8p;u$w3=PQ5)C^&4R{PH
z)eT5^ZVi&U&?Ipz;T9VYi<na@G$4a$=@Bq52`BY8={^@bII|f%c3EW@d?}D`*myoy
zI-2$!6ZZR|+Zzk5l<hJE%}-te;rqRMGYp;mM_~njXWztnThDX#xWl`!BGt?qyVcBA
z%(ZnWFaz46;ZENB=B}<i;uBA<{oo(SzxTunLD8n0&oNw;m;G4!&*KCKi-Gj#za`os
zYCUVci@NOfZgX_WCz`sU^*spVrwB%*L!!haQX2PkP+j&Fy9q95VUln=V46fL3JFOM
zdQ&(1>nak{A@5ZXAYK%yy(@hF>j|PO0f)i!#1NJ#^349^jny2u_i2>i18Q^g^ST<`
zTy(G@rGph_Cu5GQoGM|Z6|78g-~CWYD7*QqWU7-nOnGd4$o+PMBCxaaril>^3Y5sj
zdcfwjpOl#Ut&pj!=pC61G{HPD2#u@Y{Ve{5ov*~EQhZ?p=S#R3vb|ofCFdNX`2B6~
zg6Vw!0sjyBVdErHzS8aysVO;byKl7)?+ISlk_e%#r5&k~NsNz$mrNKm_?Wt^GqFd?
zzZCDOrT&_U4&F0quoy3y5?yc1B)BAgZ&hib#)V#6!nVKf$#k)Nh%_I1=dKmJ+!~Pz
z#0UQ<Ws?)ZZo#<EvT}0CcHadQ<~mZ_JL~h=D#L!Im~g`bt7ZTO_LreY_KTL|lQ~6A
zO42{aEkSu=%SI*Zg&l+Ln52XE9m|n)vFS=LkDQudyV+*eD+_FlYZy%eFM*y_9b&&_
z(`=K0%&Wzfqnh6PxQ!y7D_23@(P#Ab>8dG+`)#iq>#H<p8F4L>`822$lpo`4aSBwm
zFONnX$YV3th3pX#^!Cw2>n7)sz35g34Qt#R@XVfCZQe%{PBQKnNNV~eOpR$giwL2y
z6HA!TB9xkz!J%PPHBAdF#$jH0pkgxNf3C}4LzJ0&(+JA72?}9v0w*D+f(gfs3VLMF
z#Et2hQ4mI%9``?$>&gII_$yXHeATm+Mpki$4kmA?Mx<_+`&KZV=Me3!*YBC5M5#X{
zA$!{dGs0KoJ55Ihl=H|)+`aeYjyV7JV<mE=jr^kS&?usYa*j~gw-jQp2tk9Vlr+}9
zn)D!Pa%IT@uNnwCuG%p_TrF>~Cp`CGtJ_zcrkp4<c*TFvF2XsZ#7<D9cnSqiG_p5t
zf4)n1j90lysc`PiRkA^$$<YB&FCD8Y;l(z!0uh{+2PJ;syxBuC;eU{UEwo{0FHflx
z=4F~4wG3;&hWV~22>Zmu9=4E~#oZtCzNP_Qe{=#`vkzyzMk?1YeYfXU8c%+aWRJg1
zRPVEjv?L?qsecfmT2df0KCaq!COL{$EkXVkR_Z%1F{!(XN%hl5bQ%e8x}%@KINdI0
z#0&STZCzXl@bE3i{(Tlw72Aed6~dq@maUo+A}{yi1ufu~H{enz_l>J+mLefS39DyA
zMThpD+h(_4-HAB~?@wYS3u%At$v8;vDf<0&MZZLS$?L|xr9=>#yxLF?`Wr;k81P#y
z&^WHcW-0PZ_XGDi@&S-Oc6YPt1H4#rzn!I3!+3975id3WQd{1u*}zBOqh1s0RN?cm
z(x<j(SaiLlApfAySRP8MXwdn+gjV=Oqj$JltMP4Rp4f+cLPfq^le?|<OUb6`M0^B6
z@{m)S-J99UPc5GD&Nx1_GVIqTS?xbc?D)BLnrIcq2cOFcl*T_3V&%b1g_sOjf)y^1
z&5Q{Bza?f)b!ONG=E!#Ku7q}t&JK&-grjq)hMAO$1D9P`+;cm>swBv5=ZnmIdTWQ-
zBq**{c~VLrqyJ`Usn*u;_2S5>41kNF5Gi4V9Rjwt88;lwOKR&Dxcj<UL7Be~+0EWk
zj1b0$9(eX?gy5?SCGrzL6nEJjmGtR#u#5TGD*d~FLI4;$>uCr@TGBGi^8bQ6U)Y;9
zS~h*3FQQc!d;<K&vc{x|WaMu=oraHKC?!!+Xnnj;365^|bK?xRx>ZJz9?}2VW)kLN
z82^CAXfmUH98MyqBH3ld(=5<tGGEae4jD{EMNww?+^R^Pv6U}%=rp_NzIJ%bKp{<_
z8fSJHRn24jo2a#=cOBaqb;l;uui7(v;j`nmyaSQ=;6z>NFEM<Cj`(0hU~`9<0;gCS
z#i9HMuhv~MRwCHBqj-*Nt9O{o5gNP}|JynG6`1a$j7h3ON7?nn+l-ivVTjI7=8>(J
zIhzM{zx4K*?xUeaj?4_Bl>xNt*n=89UdQ_pk~ee&dsi7l*nemMP1|(4eKb5Q%CXEj
zQM9>UlowG+z#jVZv~l|Rmx_(sEZ{r4@vC(iZaB=YWPTW9bzoiL7&lB)-bQ51v;8(%
zpODT)7Aw%&p?~nk(ir6{W{!PZ`?Yi}x|5JI`M%h2V1RZ8_$I$n?-S4)37*AqL^tf<
zNM>wh$jGdlY}4^VtxB7p`*bx_t>oqVobVl3hl0cyn7)EL1K-SiAF+1wqv5@@r+-t;
z$<lNRImn91SGBSdO00t3oKE7Qj&ShA#HSn<oxu+3M|2WGJ)GFB*6R+}<ZPGhIOrW9
zwJsHV4Nc<V3^h&i!@Bko!jXKB0#6pSt6HgYIR(%9{1`^@{9Ww-DTUYDE{Zx4%YO55
zkuh-hN<^y|&6@2%;JqvI=uQBkfcg|^Q8?Hsuj2LG*@PPMBu^~X44da(_I}>Q4(c#`
zN4w<<=Wo+{5K^H<hz74st=9Vyv-t=4EqSD(tWls_B&IX-_0sn+LuZq+aIm(h%@1-&
za4JsM+eYE%oK(5>jN{Gt;`yz{3Ep-KM;W>MD5xOQcNq!P5<MO*5ijaBVS2ruXFksp
z(S>0<+KmnsqRryo8|x9A@s2W0sftQ*Vq1SScv76X{+R4aKaDCECO`uE9Qh$;sB#7W
z{is6#Q*Gpu=fG1?hy8H!xVB|TZ@}ry$f~tCCF6;_q&8~v^L2RX854ko%6UD*@Rq60
zdB~}jYD;SF3W1TDA0771IRi=HuSUFVur^ID$kn7^4ZTEL$hVey?^tA=IIiNnVr|br
z;|d=^EWGP_6PX*bB~8k3cPcwIh=$Lq+}CTX*8D?R*=Ov%`NLJtUqv5ue2?NX`vX(F
z<%(M#TSIPv#Tqs?HE7>a*$xX8TrG;<6QaJgtOX-LBr5oZ|9g)<yb}InHq}-H{<Qjg
zVy_k=8V#w6PH{%t$7Tw6vc(wkUsS**{hh!kI67?c8%o=yz%|6XYs<lFJ08F8BPq82
zh*_5eE$s6jikyC&!~5IclpR$%du;<dKEt(*|2nI=A=WmnjW>ve;A1%-*rX1FyT5s{
zq}F?!O@37PAfjwzQ&Equ(Cia4mcvaEfAp@Hh_pY7&y9)p)Msl4|KbXZgyRh1>e00j
zB-_Z&lO^jTyv_9;EU0{=;WT*{Zjk)N1O}vfe=oHmc2UgmWXO~)N+DPJlXlCfGwt1i
zREH0V&mM|LWtQM)b2=%C`}d;quKt6)^;#<_nf^n}bm*=hnB4IYh7Y@yyM9=@y+iP>
zt+!kit#lalKq;M*4jpwnUJ@wbX+Cap6KosZ=T4n~nRLQlV*}ITyYuaQ!j`&X^;Lff
z2-+KG-|J<=`>5lhVvWO|TyjVBo}&_@*zXhKFRXy2ygUip)g4Zfho=F2z2flRqkGf7
zaJQMpnTX_a1sR1De$-0RV(k2+hfk>8gi@9zH&z<P&W3tI6_ubsPVs4*n;Co@Lt#qQ
z+UHq$uynHq0Y?0ZI#$oYuS#L{pJIijbatMjQv;3cdEb&&zEA7Vs%7=q@)X7+mL#!p
zB&H;ssLC}!-`qi@hSgi)HA~&!2YlTeF)9TZD2Orir-H5l9~r(sC!kmPuOar7P43)x
z#i>aTegqy0@7If>IriP1$6+kwCd;Z5EYrmQFv4nF>BySB%?)BNCN#WD09lqO-+xYl
zQpi+xEJf432<8fjh{G7m*#J}UCqc@JpOzXKJ@C6jC<7FGJ2REqed|gHF8X1Fxz{BG
z)%yEo@lV+#g@v0g+M;PpIzsD~y{ppJec|vtyO|YIUFABv@>3v317THy!ki+jm8^I6
z9)t<|%J3}So)6~LASRapV)XpHs~~LJB5I7AM_0Uo@$`iuCIn12z*50?8VFyr%=o(d
zm-^rGD6^9te>B5`7tg<ax%`_D3F#t#SVSq_ff?QDCB&RjiwtGhT65xWvjCFwvTBb@
zGU6UU!G43HZSyQ8lPW8I>lMoV$2i$;f3r`&_yqLpX(?W+yacIN7eM-G+m3V3XA$xp
zFtM4H4RRdhuMR$qXgNF;8}N9oljQH)Pv^byI62p9$e2rrh%Jah+h*{b5YU{TVWrIX
zzdN7g7k^e!)OIZJEYGz1JCO3b8c+UmA(-9zs^th!{*xPU5$=Mg%W&|FVV5e7uUsGN
z*t5yT47fz?9Q_{Hp6gbOo?Djb-CgtM=J4!D%J~lx0G0Ki&_H~YXD4F3O?fUE`G`4O
z0gPR~^dY%yvJ~;i=phoeVyLvF_?_keJs*$oC!bGz13Oomn7J*C1-=%=+W9B%5erWb
z3~M%^CV`>e**zAh$(CE(&P-pGs2{FExopbi*VTvYaF55+M%6Yc6riiRt?0X}^MMn1
zWg_)zTIm?H-czsP05;;c$!-!;;Ce|C;;aRpXa8&^(-HOn0Bl36@%LetV+Jx_$4egG
zSydOwm%VdkQff&yb<B$q&Jlr?hON}>t?$B*=)&_}0>mNL<O0WE`3#nUN=P+tft9?Z
z9s~%Emu8qU6Z8vLG;55gZvOlAFB(I*eWiife|gNt6l|9^EbuyNviQQ+$nc4NS3YHC
z0>C~7XbmpHSc5E?y#{{qYT1#knoIFvmj`>QjBQ}z%mY29C@YZOx){3`)V*18AZ0_f
zE4G=LYx}`%<KAI}WQ>iHooa5oL&X0l2Ys-_PuldcL5{f#<kNs}fbKb*uMRc`mPHqT
zE#9S3c3o>{Mk;RG)#nwHi$jyyk-)nAeZwN#9t8K42*Y%<FURR5)w?*aDppv6c~H0O
zcLPos7%QIg;3tzinX629SwW>A2QDhP5oL&L7|m%e{QMGT96h^&vS}GbC-=<r?JZNv
z_cX%AsT~JDh+2dHimFtSfK|7xI$wm5oZ`{<JAp%GX%63CS3jkiAP)O$;{&jk@12vq
zaVLw^3ya0A!`OFCm3_r$^m&UTFU2l2;+CU_Q^j1OIZle08J!<kGkEbymcN)i7fh@M
zT2Hn}QHAF<JMa7cojiN~>7a>77Q$-x*x|FlqWO3_k@S{AeIhS_+n|aX;bC#s&+B^-
zb}UYS|JfCQ)K-@rw{eQfI@7M=Ohoq250-G7gZiffG$=t<K9@ZTy0)>O8NH_W=m3Q8
z%6mzqp*;>aWBRLoSZ0+Sakq!g1lJdL_X8G9uc`^8Y}GmCKa~5!vlx60gVIlHZAdbP
z-;o{mq1EL*{_rGbP27re?Xe2svqgi*4&ecgIgc`Z0&26k%Sw>`1wqWt@17rXo`4LQ
z;0I1kUVMc3w^j7iR#~R$%a7G}hj5EKV&LWa1@No6cZEsou1y*x6p~uaXk>m$z~zZ%
zp0)p$3V}t5l+O^5ENTwz_Wg7P(!y}#pDkKGNK0d62<<E02hLID8)S*O39f(R`zw(c
zCJ`PdpZ{Gk<6iP%Z@}M5>~Eg<bXe7|KW{Rb+yIpe`5(#n&&8$EMpF6NC54Yv!v!TI
zX3?66f?kiE8&^-^t2<HqU0ZNL48e>*1gdAS4xY0NH8UkFu=knZNg=M{Tfv*!Gy08(
z^^<0M>9n?&l;=kYwOO&qHz-?S>H9V}yafyE#Npx&@c}NtAF4Oxie7VWJ$3o2FsR89
zqkxg5?eh3(L*CtmG^5e_hH2P*GB!y~oCB>U%3L@ycep5X(Dz(J*36*?1i(^>A-Qs@
zkJ}<;t_TwV3~cn!zs8AGwvGw(m6;aqc)e)#Go_w#RsV*ZBkf)r=S5Z`UG(<mNwd&P
z9Kky-h>6fARd6h`)S&&Phf%rWjx{3j!g-US+GX$WA5y8b-*&Ed3zVVwfDhJ~;zW;O
z3@DO-)=Lw9G!uM$;;UbvjF+ondZ{Ggt-=n#C=D><6R@$0ACibX`WC=<SQMFu<iJv@
zoq)v?IBiVbmXEa%c^WIm5l`G1q)fh853<-s0T`?G-t$5{iS)$N=|^rAfU$L#&Pfaz
zLTAh&ygRnhRC{bXH7Z~PJ|_$^Iw%R%ZT)9QrbDfxKYG$#4i8I<2>NLxU60*_!0~X}
z9&WUrTI2HA@4hS0R^K~Z4}0}o!+r56!^0<`?IFJxSoWf24<(jOF!rKU--sTWYwdBa
zIvyV;Oym7>N$L2_!#-K-AsP^f&({Pmn}lRf7B<v-cWWS(gkFAfM4Bq|z;IESUtK_P
zXiI%s3>K5V(`7*Z=s1^Sc;|_2Am*V496QaS+Hc0*jly*Cns;mzuyem4<eIML5u0}T
zWXAgNG5-R{us-dB@ev&jULW)UYLV%v=@Ygrzhb2^1VMS<^nNhA)|nR_(<(d|)lUwH
z%bLN@0|eU}3ECWjsMXdy?kJ}S)9q>VUBV6+5B)0*YH=m0kR^4da1Di}ts!bL<WC<|
zIS1Pci1?V-3A|=iSZ(HDjp1;yJmEst{*tgay_awcy0K<U;py1GW{p9%PrnjN5odL{
zRLyT;C>(#jPV1RMXBcR=Q-)IBgdQDscHm@OU}kzvh4vi)p3$Ev<mZYk&?RZm63}2d
z36HB(QvvngD_Tdo0B@PU_OK0iH3$m}@BUh-aUK;UMR2T=MCwJBa#$?vpimG`Ad?;U
zJdg|%dr|<l?g(p8hg*8!5{Jt(s@X0^$j&a5S;K~<E|^el<ae&tDwGuW6!o?JFzGON
z%pLssla(t3wE}ZL@ihkqq5lCa#Y)tiB~rSFjkXOmj$6s}*sKp%lVnEe>WwUmyldnS
zI&Z^9Ypxn?_vMvQqR5Z{W`GL1KrJa1b1VSrG*qp!<U2XF1(O0UfLh`b3oY&=JUo{0
zrlhj2CJU0#mu^b#-3t7Pag<K#1v%hLsk00%1ni9s7gkk0^c4iatQ6E8=4RiFs}q!U
z#ZV@dK@FiSCf-3f*>QLPUQae*zoK*Tfs`*nQhN|Dz&^DJ$wODY%pl~t{UYWB-7RwB
zuzPK^V|;|#TlBYR5*_+Z*q34^y*Tc}%<_4G%-KfTa^}fF8-6%pJf}3k_N<YAO2e;&
z?Ks_id@?enpclTJPb?YI(w)R3y*4HziH{x7BZESl`y4H%Pi+E_@NhN5I6mR1&5(>Z
zw}l~Oq+Mn~$E7Cu1(Dx~2nfe}SNF9$cOU2}#d|x#D8cB?tn8ImLfPAknU!k+n8L#f
zXh4{H4$HIuu<J7}Qzkg5Su?`LP$WqGy@IXMLsoY8{1mcwXugUgE+6H<YZB(SS98>a
zV&a{L-TuKZ=Vq5DWXQM1N91umQKPW|JHIAXz>(oIdU^JJ^r;ONZGSiB6hnOblKVDh
zzxQHF62l$t&Y@5%IMv#9)nfKzN;n`5|I#<0v<fMJw-_&*H*mg=9}rsg^8<6D?WXmr
zh(np`-91StXalB|fxeyc_fCT%T*tZv#s9Qk;8tRc^af$U-a7b<-lXp@YtM=|EI7yX
zmg6E=7Zs{pLu2hoy(X2I!+oV6dFWhw(3^h1`$HC(+XQkS&8^gI6Bv0oRHDrJfvyQ@
z8a*hrt_FSnu^N@Zmr(q-{y<|?2(#4rOM$zs15<aTsY=Nj?w|c*NOsYPW-!*Fcvt3v
z&ILIC>)Ka~;hIyYeawg)4X5IzX8-FgTQ`55F4o^ZH<<UlNRHo^irow6lj!u{YdWpc
zDq0`hp)meg^8BqMD9Rdkyj<Vr^DYXqpJzNW;+)ggjc>f3z@HHp9i1<|c#`|p%X!sg
zA~Q!WHN~^D(iijKoDw1Ioa)Mv>FD#K^6t1no+-6V*=vGAi)|@SB(djVJxtDKASFB5
zcNa_MQ}otuIVu6O`m_LE0WPhAnzI^3ORwzY7}^Pk9}%x<Y?(t))SB4At!j)o?2#CK
zKa;(Ko-y&$50ary^r!uF6$dgBu?{?%$b^ER=QJ6RtR!`xbm@3z``nY6Q{=+*C#lTr
zoI}}VU-#;}sxNqQoBNxMC{q<je(yDjR9|lHCr1J>JM9*(pIEUPqpOA>$X=q98;FDn
z<5*dAE7RmFB4H^{=g5@nVxc*KuUB_~6!y+~=FwFdQdIp}>$TvFwC=1!m0?<@A*YEV
zd6MBg!@uPR;L3=7<-A|oiJ}*4aoK!vhyg`5dL%bqM@y(l>6s$SxI1=wUrTIkSJ%1&
zXg>@npoK*UeZEapZ1ENiQ+VStl)_ol)?zES^PbESK$R7a6apk!E!9KL{Gd20)rw3F
z(346d(Rq8gF>AZ7R{RG~*=K5DwdZJ-p7QdIBb2JNXKb3{D*^>yvwS{wPc!@1W@Wbi
zs5oC(QSZ{`H-53gK_T++L@buzmknL<jPQ=2z=zy!bi_=Dt_VU>Sb^=Qd`Ty(b^&9v
zS5eUkhS~kM6^@$NO<rA}(1wprZJ=K<rgs;7sECsJBg8m}w3S&BSeMcd=E!B!1~%u{
z>n9zXqb*x-CmWsw$fmMqeJ|RK+}jITt~0q4Vfs|V^RZMbZ203ZC5sWc8Y)vTmExX8
zj^m2)XWGMG+piY~yh`isD#jvD(W&QfC9NaWEK~*sUAKS)%pOL!S;BS{^|p(+B(|Ox
zH+gUF-Oq_X8`MO+W%^iT`{&2RD8AUQuu-V3?4LZQKTOrZf!e^%lB6*;z$PIP{yRWA
zft$AGHq$p<KL?_|Jjt-|2K(8T0Mn&cO|y2Iy?-iZ?59lN@2me}%llKKyC-5bUcU`{
zZSA?&eS4CCNvjF;aod9#dRweIBQFs673)y+10y4&7H3T;Xl8vxAD5%aeg7+H#v%u!
z9Q7paAjLws)c+d?lTmqAm);B~c6qv~#O+r-1#2BE)m%CH_?$eLYyPGC!>5fv>&1l*
z3zTv&nPQro*3*KN1gr6qb{?x0g;9&KVEgI|jry**(WCz3L^_8h7v`azfO{W~q}QpR
zwDn9~^K^xqzHft_pTaJ`V$kC~VZg=Rjj8{M|7FS@hrNn)R{-~pL6U@{qRiTjhE<sJ
z(`eedR@ygy&A{jE4zfpa1X%+=5i3P*F@5Av3LqVtb32efW)i>rzSn^9UEODpr8@G!
z&rLN-G6+^^%wWS4JsMt}naf3Ax5R5mEgQ-IL{lLRJG>skRcQ5!<R|q4NwKj>oCX~B
zv4^X=>0&?78@m=O6SC=s6#L&M4vM>(#}H92gc~!f*7%o>W=<d8<@Xrfx1C_pn-*XL
z*Roe~sr#F@b~}1hh5dg>uR#_S)7lKS2-9g}^6D<OK2Z}6l=dJKA`tU@1{G6v4Ud}a
z^b#It>ns8(D;vk1pzg$>=%vcL@H{?CYB;Hsa}w7Re(k?a0G%r7G&M$h_jN;Bk|`r>
zzwLa<sDh<2yND`zyap>O=qvTzHfcqbCHzoB-emqf9YDy?ck|MN-O`bR-?v^<>`;a-
zO5N694@iw+-W(aVztAX&7Arq1JkBTPaF`QcS~#k_t@L5~-A#Csa{Fm+zQOS-%0`3|
zo18UA)a|7?ha{Ee37^4x_4$v+6G94pS_+!&`RQLpiu#{JZzkx-PjW>p16hAwMrX(L
zMO)7Nr6<=>w7Ts=A!7-NbG*h4zoR?KDVgt#D{iVEDV^9<)jZS4*E(6Ze$#L~(Y*Cu
z!93r~?NF-4{}}`F(Y%|!h6y6~$!*z>-4De<`fg36+gu%W)Z>Dni9I=;zusnw4y{~m
zAvx}Hs~I_gs94GD-M1H_%e`x^fS#WCPfa0jFfwZQV_x!nG3_)T285>`a<Pi`N+<|<
z^PBBf3%J`K&fsj9SDeRxq#nyW(s#ARpiDLZPhQjW-|sGVr!LxsW9~sKa`Cs4z=Ivk
z9cJq;gr56Q-I!8byY${$hsAp4S(~9O3z6p*DIK}?aJ*>uD2TCovAs`=?=}-_Zo2}|
z|9o6^Af6mDj~?#fHmq<FUvq|nx4*TniLY19^D+L1wV^>#etyKvG>N2lK>Mz~$Si%C
zd~JU+QSN*8+O7y84RpcHab2FB*4%xg_7+W~H`A$5BNdR=?yviSqU^?*=RuP~;0|i%
zcn$+VEZOedn#Q-rg!KM3&z+uwXy;&kq1sQ62H)rrS+d`n+sS9}+3`xCeQw?2Up`*9
z8rKYWO&Q5P%3iM#BIa|`>3$vSP{KG&7evhag=%jM%%{Ahza@VCp4<01or1+Xp$N-h
z(IO_veuS_$v+Pzj5$61srC+K%rH<4`TbY6Z@W&G|&}=rr!UmOcs$(sZqV3G9u>q}J
zj4xzHnVbddp<8y7H1+HIcC3?U8G?eGyr#QV^f<?21>JX5E=@OsPjx;s9AZPRT;EtI
zWdh_V3+MfpgqJj=2A78yEbYM&RT%5Hx$KLu7Mn0-3je%2h@E^>a<ZRL#WD|6G^WC#
zV=Flky__1rW+#QYq)xufyI9&?&H^yl@38qzcv2PJYnP7vnfwJLVXTPOFUxt?Le5v{
z$FE9wT@*+0gVFFyVTXtSoM)F)$8{U*gVf*@n*D+&EZ;{@82F0IR6%C2FSr}+{-j)e
zTAG<XVo3d{YiyT5_XfpzO%<&)pwUal`WfEKHAP7qI?~J?=ZCC{ZD4e-_Gl8SFKq2>
zQgQaY$S!JpaWq!l)Ov)#@;I;9nv#I`^2qw9maa&T>F20$w_}$vy=B?+wT$>=KXGo1
zGH&DiN8Mhlm30}1ss`T%mq7uL+JdJa#8Qb!o>sb-p+L_;ot=Y0$J#l+59#|#v$8xr
z)>PRoO`YFG23NqtwX&)30)fAcys`0G2B%rSPF)|tH@CHc`?)x+L|%P8c*3dQ2M1QI
z_4Gi^f4OmpHV`jV$^08I{|RI3%H%A<2b;<?M!ABu=V5e_pAz;#nE979_tSG1HY@tU
zw`25!<y=2W=*XK81B9wf-Z5=9Fy2sb7n~4zPzqa4$9s>z)A{Xwl=zS3KKWg_@r@4{
z1>>@h{8OU)u<zNoQxj<rb*LGa9qo(oBi>YQyyLI*-?{OBMrJdf-3O~}q#p%CPBHv+
z%V!8(tlO!L7Zq=(;PRVk#M$+FxX<U3Mj_2z6Vh^JCR34hskL#cndjJ;cWW=d7vZhV
zz84EFeg+$_ANFb0%Mv67#|RN&L&;$~4RJ2hSrEgZ@4~Xw+VSGq^%W^VOMpDe3`pfi
zxT>q#Q^~_bhPc?qV|_6buTD~oa<9ewHyV|?=?h8GzL7tmg~dA>BYeHNs=uRieT9<3
zjz7|5iZsHkez#EQrITY;C89fU){TNZzxyEc!!;%kMe4c?#U7N*owU4*MWuJP-TzSu
zYzEQQ;1+OLyGbgd{pJwl;1>>tAz>E2bJNy=T`Ke(dtcl$I2odBk6u8hYLtm>AHMi7
zZOvlz#o&g|{Q?^DW_|ZsZ3_~qE_9G=O%Uf6SIwg3S-9Q27^uv}!|EH9*J2Pi9O+eC
z_3UA$24Q)p^}RHmkwC!$%FV7BN53}aKk76qk&+sp(5kR)BH_NY)#tpaqv<-}27%L$
zz~@Qc4iTDjrwW)-o|{;R<uiX8Xf`#{#*L~t>i1`2-l$|sj7KDc7Qni)5k42&B~0~y
z(Jd!#r}xHJd_F@KQ~aMGb#-+vESGf7{-}%F6&+qDD;Yyc^!m?NLD!;6|MOn}iJ15D
zx8}Vw>9TsA>Z17`!abf17QgI4p$*)gcO>d3F((l<&ZmO)4%aN-Oha}i1}KY<Ima~v
zX1=zB9o7$d)K@u<XXT}L`2Igsy;F3g-52fMQAZuCW1F3hZKGq`w(X8>b!;aUt2(xA
zc2Kc->ivJ?8)uxmx~hx)?Ded@=bFFyobcFehLlTPli3ofgd$+kONFgsOmFF<Ahc<<
zfG)g*X7JgbgejqIT^>Pe4y`0n?Lx`SEzd_TyiN@Iru_#%G$4NX#g9!*dY{pw)0v&%
z>zFJR%=N60ha25@tr7VjJa9>X8m?cGL)TRHFQ5G4Zh8O2m1ch!_pGVVw>$v$$O51n
z8N0dr2X>x6naaqsE#QQozaaLqSg|aBDazaO<SF&SUt`n#m4zY4*sNL;xjPv<k<3JG
zhF3uUn8{T#x`-?@=pq1l_0Mp!DumwRa|-{{S=wc0AjE|=3};y>Cz|>@SfSrDXtxCz
z=Tdw9{p$*}DdFCBrG)DPkEHCjzJCTjL+D3<v_F3Gf7CG#(z94P*r>6;eNWv-1?WAc
zJ4~8oo@<2jr0xF|A(_B0$LtOEZ&riVW&>%Qu33=uE={i$n*c-Mhuf&0jtfL1zUK0{
zRidre0?a`dJA7U@6NUvIKiRr*adBmb<B5C6S7Nte{sz2}&1ZnyP7?k!s~#Jb;Tb$5
zGh9|je(4*@IlYJNrgX#j(#&|S_xyCgyI4G*dq6-4J5ou5bQzQZ=91Cb#(5@Hm-JOX
zv6BtPL#mh1E#j{GX-aPxcfW-}fVJI-`*G1G<!w5AGNL3Oeper`hMa<PzJF^iM?@y{
zceNNt!<8FwXm3frN=3CheBmZD?hUrMeMK|IX`92^*`{&Zr3-1D?To$wqFN|%&e*Tp
zT|mFM0r{^hX`W&!-z1fhBmLJJI<``yMPkZ~BrS4V!w__`ml8B!v*Bbuar9rmdU$z+
zNu1<|XnBX>Cki=9xHTiV>BMTYaBiS7;4f0m0?l~o#nvsFOBjP>1rr_E=wL7T$8+}!
zowCqz<ImpfQ^Zb(?Arn~E=^;0#4gVkn11)^%1E+Ms_&mY%Zi+~bTbO*4?U=k1%wiQ
zmeIn(UA#Cc5h$@=HoKf9LIJ)hjMze6x27E9<?3amHcM0UbD1jtR*CAiw=L7;Ll_Q3
zYU)=<^6xLVdr@sXbn4Yxkq|pnw00QVuP#e4m0BQE<G+=W4SowY=9qjLFk5lgE(Bj<
z>&sH};p;{xVf4>s99EixR!kojcj~MLGK}`HIN1yhQ-*DfY7%TJP$X}v?WS&0!1a|;
zK-iM*Yew|>IGg@9d<bY8p7Gt|c7*9K?Id2u+Q)rn&hGR~x;mavs?=Yj9h8KAJZpwF
zzku}{UW#pYcyw=Q09ZEc-R)W!%!7c0gP0AG)$hCkQIjTD<s=mk`~33Sx}KUD(;RJF
z4d~DiIMiBX6H#NgN*C)mA0zIKc7qZuwjUy$s1gn4MGkFDP=5)4Xx3w|eD%ao)Yic)
zr4pcRmtxw8QyTeN(fSG*u;o$sOWEJ^sOdMEk=a>c`kBIEN1h>GZw)z5oi3n=&ebgb
z6tAsQ=;b9byOBy59;VWz6_BigvaKVi6ZaWG-bWrkT^rIaucfVJ-XTk@wPbLfdM0b|
zCruV-3Ac!CJ|Zmms3KR_^J2|~*d`4xiP64^l#=hatzNC0N2gxVw}lUUc5|yI-}G!j
zKBus$k3BvnMj^MnF76ADHJBMd`n-0bQ!a+H)9uWpipvz~e#O@h00M4D3QSNk=vR!B
z&n80O<uzM5Gz>C_Zy2Ko_#+sZY=aEWQAH7#rrgKM&inl${w?Bcp=y5`l*M8J8kQWd
zT6}%Z>xl_EnF_p5%12;tLF0-3GbJdlo<HG_2+P4TpmZI|TZT*uohXLJMr$EZr!LRH
z4l~>5a<iRQA)f(Ld1`@n;(POQL%&8=k}y{o2UcyL&>Gv)DUP54eHZH=y&5k(@Vsb^
z@@Cec=~xg0)ED+Wg^=rHu1mhSg<7L97e_#ck|V!=L!KAF;p^Mg&K+Vo6Gr^F4bf4r
zw3zarpPCz*Os4a~=36Ytfnk71S5Q6!L#&a69)9=^SPQ`ZoR4rAu)FQW;OM^L0p7F_
zA0r{17#HErX%Yy}poSD13;-#C;yj%w8FZ=)W`6!PJiTJa3c1Dayc}lBDe}t!@Au0p
zE%;YMOUF7YpPyZHB3%#L?N*$zgaSdOvH8;3>UJgb(iV&zkBt==oLIm!CzrGj(cWXi
zYtO+~)1oCH2%V2M=KTUEI1utD3|x<(!jt;>RzQapuhLgDVdZ~r_w^ulpBaQR)ILgS
zBSQS9W?)AbQ`r4XqnT-%hN$AS=tqTcZ+$=*HfA;wVM(g2aoC@4y4P=i^b|4jw0@~;
zp8&;?AT!E_&20i$)K=Q<Fcux&;9}q0@M;7dvjm>4i%}hCB}kj(2kV={v>-RUtYCCk
z^B+1e9i5PzEX_%0v}MQ@0XF{i(5HohVCml$){m5leelSy&lGZ*vViaHt&kD4Lh?l?
zOLq4yb9-adod7<SY8h=o73}{&A^nmZR%qMk2uhoGm>odYge&$mQ#-pRqLODSMbu!q
z?o3yL^><VM^AGOYTJGXcs^d<}S}Cr<3^L%ha=kHLlds`-0a0^jJ<4AooIxbuS9?Mq
ztQ=PJeemtUZmeb9%sFilZLyfuu9>jU=BnpOV+q{GQ$xhcyDna?0-r2(JmuV&SJkTG
znOebNY`J!#JU-A9Bc~sGW>tW&q8IE8j$V{WV<NpF#S|iuR^X8zjT{A*(YRmPzyrtD
z^yuNVdEi9Kp-{dZnbtX4>kD=^w=>d@1w4+!6)1(*f6T7uBN!3dx9reA+Mntr*T*%Y
zE)_K73~`jNIKr+^RI*og$ZJaGr;X^`2JWO!T#)a|_rHm;wrNA<@`Ex$61V-IL&8zs
zB|NvEtZq=S^HF3q+jvke_Jycd@3XiE@i~EEuTLiyCtY41vqOGV1xvm}&stRS?R69E
zy*Wkm&l3~#Ihoby&bh*hi)wIBaF9rJu(PM#kBS}6u9_@@eZ4kq`KuH7ga`Ip_&6A=
ze7m%3<M?<~O&C{vZz#ms7&~0LvQNWX<?lj9!_i2VGjD)@+&S+YJydK1Z69J2oK!Wz
zfZCEL(bPtR!YFKJX+^gS;c@eK1wHTHzmPdTp_3&3C2+|wS0;*rGOZ4q=G$FmG(jlk
ztBvN_To*XzNON7J#Qr6GVgUOW;r8k=k+TR<SU%x2TW5aS>Q0xqt(=IvfQ=dpoZ#Va
zjM5OltVLRu8>2;EqeHN-EdG_zk`G#0tS&k&yRAU%b8mM0JwoaARYAhluJ~peCM95D
zP2daRoZsantNDDmBKgniiLZVJw^dRv#{F0H9MFcX-Ym4aQ+n(<NhY$Ao_@tGRU(8g
zyw$+BG5BFl-~$5QioR`x8>v$r7Y{ll%UDP=hbGPA+*J)UKJ$uBzsElP_C>*X8M1((
zEES#3bGb4{f(%xRT?1ltxlAmPkBq0T=S8Z=iu;4G2e*2M6Blnl(3@(vf?B`j!A%%_
z$_O<q(S~0}+LS&mTPLa~xz;?scF(uhw<+B58eTi+CiK1*&aw1RH&C1DM)qxBI1%4v
zKFmVVBVwt_f@9@(Wa&O)TpvZKzecFw?x!L0^KJFF;-CbFt@%9q7G8#f?Sj6Co9@eB
z!s!T+5tA^|9mVHolphwioLIFN+nXRWC+wKc&Ij+;I09!=yBThi#TE|U)H>7=aX3C@
z{@+LTo#TK)fV}paW+qr<)?_uX(Fx;0jhlq$&@_}zo2wsgObzxjv({bgIi#&r`q|;h
zl&L=Tu9CsBvX)*o)~^ep+iRs~R2NO|TyUb-YD=!i<q@f9L{Ht^mAFW!969&c;H>uf
zu@u+s@BD7(zeRA#vL*K<v)r@LZKoLJ-x0>8YPDH;@0vXMtLN?Gr;)wk^pj!fm!EJe
zZh!OE(RW?hZL&cOsU93C94m;&4A{d64!TW|lc^$bE(4=E8NIzKLB1=|j@u6#OV53h
z&!hqw!|44b8T@DUwov|FyWGRter=I_9d3YM{;KvXq{;lXwtw?}y@<D##kDj(yhmx;
zh#&rE57n3a0%%v$diA1M{Ro={yTtK@x<?GhE(7({qXBwWTlxEPlAl@ZXAQ+nLe!z!
zaDaHe)zFI#bKcS9GTIME$jNsXV#IGAvF1o$<sE`Vz8bU;;s3!PY+t8Zjx4CClz{#-
zs9W!j{m$)#_n&lc)rEVvS_?XX5!iq#3tg3EK^i&DFmA>fqCVml4U5La_S<<f(t*P}
z8U2ho=2rL*b(nz>Ab@l^-|W>$qv8fMgAmQeR-UlOeXCg**I6HJZfz68Z6t?7H_i&>
zW$#J9eBO%V@9Lu;S?V15P7>iSaZ~<FqqX*HY8Z?(ORNapVCYjy$AtU{7cv1%%6BB}
z&S9AF#R^N25-u`bu8>6(1?2l*P-byo!wRHUvgLa(_09inT_HYqp<n)u83Vaow`Iua
zfWCf9HFqk01}DPAQ7_&Ypbt0n&_?G|w;izuOGGUchKYh!SM+(yO>AWcXqedR<HES_
z-AB^Pqmt!l&m$65xIl4l{ZCJoEnR8v?6;_r#cP_ALVGYaz9goB>tKEa(XPle`Dg2e
zzlZ$gxIZF%<-ipEO~`T`oWZHk99rj`nPzC}=1ke#$ig_^fsuS~3fGVA!)iDb4n^t6
zA|B!KS(q?GXM8!hrG_!Ei{9A9Uy#$v@773H6}nM9qqK|(=^y?DS>E#Y7m+q<y3C{Q
zKF@LyoRClSHEbi*f)Xpi`zr!t22Y-%C*7|9zAo(7C{e0Hye0IQQ%C8v9LFp`^`-*O
zG*d}Z0cBWeE9ClJ8a;8A8H%!yUxkaqgV(D#6(csZfo1Hj03sv=TwRV+Xvc;xG}3pt
zx*`YNPEtPR;;nXig83q3XbZ;najIoK;7h5S&OaJryKhD#7zc4)UtFFThW!4TvwAB0
zO1VV~d>w%5S_Y;<2e~eN@?<L1hQe#VLi9$@({eUi<bb06S=?ZQ+UH+?%2|mSB41CZ
z+YJ!K9t-4<zipHC+NM&Gp}ew95ygw9iKe8EOG)CXiqw*qSMY#M=n`MDX4LO*W|DFN
z>&GuZSHp-`xgtK~Oxle2UQ^5O{98xd+HcbXbH0^YG{i4E8PgQE;hQPpB#7ZvjXoZi
z@bh0+ZkhIxY&jh^if>Y%gb-|vO?>oq6pdcNw9T$DBINAW<|{l$^0xmE<u=%mc$I?y
zF4u$%G~Mui&=NkB0lDsLZ+vf|qz<1*=d;=){^fnqzcN<hmtBPK6L!vejMbvQBN$Jr
ztz0V9)~WS=m}ajz-t%3RLF4!C;2{wq_vN9l*k#qz9dJO?r6IevC-?{$wQ^0s<~azd
ztmsT7GB$-KAVRci&rdC6p;!fWN?HarTx+7UwcYkyG9`hwvdB1y40=r+_o@7j_KMlO
z?mDY26g3M{cBk9^-|Z45w3_h6yUU!VX7#d5=ZB(DpeS)(ZNB8wtD9Ots`XL{qJ=`q
z`AXXjvB1<Lym{}8-CtFRQpBC%b+#fM+SV0Lhoq~GdZ$I~f=QdLE^flrfSAq9RBD^r
zu`I#s@+%ZX!3KKlI`n$uN3VB@`+p}7HEo1{Zvq|_D!IC?`y$@Me9CL^0ZQcV7%<7O
zmcp$LTZz&mEsv><O^@?RMuZ~vv1^Nid;oh_-KK-{Co@0EV<*c`BWr8E#%DBvv`2dP
z{gP1KNb4zN(NzvChhyc-HH*dsEIPGp*FxVe>8HL9!GCPTtyPpy%z}+(9Jw2vQDe*A
zIWYPHhGG2YLOzIVP0qnr85mfb;|Kca?rpVXUUJ=4Yoffh___3wC2Riy#5odwSRF(<
zlS|8LB<F1(+IC#aklDV>hhn#czyUOCOG_02ons;wgY#vI@rM8d7q{0uOO%HW_A?Ef
zm=C7hUO8E8#*%I)Sj*=eHPtR~MOPR6>IW7tZ-4$sDbw!f(CFDB8an!RYr%vn`F1p!
z9r2#~LE^q4*w1@ql3r6FNn}G;N+FL6xjM9}XIdp+TLk18irxG`UGobf$Q>?i(eYGS
zeka$hGw9t#oL4fr`U3fMOFT-gO~iA^lMF*}8Zu9JCb$`YD!8a<X>+LB;meoL|5L6E
zYz0LSe$?FX_3pUT<Pc}6Mr5(0F#tL7)IXrqu4}+`%)KJpdhM#g0y?;aC($Y~Pqi7D
zhIV?eCK7Qg!6g<JD2rCEC-6J35+~=d!iWhpGd1MsY8Ek6-EJz_dqK9O*)w1#xY6pc
z&AGW4LA@UK+5nbJmC`dTDz<xlqC;qB0@_&*Esu)5*j&c=)*75}K-+E!AWlJY%g?J0
zu&~r!0A|*bjf+;{_eXy@cC%StbwV@NF+vv904x5@JV9jdUw-gr^$_xn4OA@8m4l@)
zAe*Sp5+AaW-%8nM5uuKUIr<VJ<^*%7n<a;^WBg*h6%=W8cIz&pmE>`dAVGcN*OJDi
zZp<>QLn3Irs~|3xLuahrJ89eRKE-E-bh?FI6<P%C|LN?myoc!gypA2C*qnzqRw}q~
zLp9#<g*(15R-4WJWCk56e2rG&EKY&aP|bZ0=p3uu9lKmrJt+y<SUjiT;V&!+sy<l)
z5BsPnW%Se1a<|Cnm(x2(eIPxZ2m1PcDKs>+>yCiHDVV~M^>{cu0T?->PDvtg+T62*
zNu(5B)bUFyp)6v;sx;fKM)lycW?@fiO1y9R#wMXF%yxFI``>Yzgw4R!F#Wrx&W=V^
z__)?47Eznz2`3L|aZ(!TkAH)?iQG>tS^iD(u{f2@jKZkhEajOzQn|`l9qATPpQo08
z(RTEYkI1JnlZ7*==I$*M@g%ZLyUoP2Ql41+OSaRQx@&F<Vz!Q@9(9jSHg>GPRtCAQ
zi~z;CjjVb(d2?59k6v!P-6ct!LEWHamu#>VBu5&%?~z@>iTp}ajy2mE(?$~5Jps>1
zaNk%SLcReikvPX4$wV4ndTDA4(+c3f_~2!>VOQ%JMxDX367yB$TgpF&`@dX}^aSdS
zE~9^4vsq*GrCitU%_RwP<cjlM-Z&?#DJRbDnJ3J0y)z5*#>M>B)}>1N%pZ{gsbT~0
z_4Qv&OiaMs{5)|cixIHl)r@r*Ie+`i`TXbOj&CJc-W=GU02F|!xF{%Eg~rZC3W3-f
zsRlBMI;33@t~B@dRSr`ILz~`4eoM#wrORTK>rvI$0z6;54KybLXCeUa#X=K5|7zPk
z)NLQUx}J|KhnHIhzhCVPi^lJ25`_iutd+N8N}q%(E*F@6Fjvm|>R+>6&or7`mz1n9
zZ@XnnWfxIU5GR<|hj&hWP!D(eWi_~<A}RHE3qY4U9xT<*%{`BY#=2d8Ji@@)`03zU
zr>(Y^FdkpD|0hY%G~}7Wn{(iRlMLp6-m$st>&Z0C=YDT)vU*e|zS8oceJ0s#chVv_
zcj3G*<;qpgBJBIX?r|@uVbmgxB3aoxkO7URv&ORCp#Hgq?-VmANX3uhSrtk+VO0Q}
z&#WYx%v{!8X5uYSFnoNcC)Ef^KoJAuKhIQc+J;k%BDG@I|CzSE$YK>51@#osL)YnS
zUDTM#en(-}x81cFab@BNFC9D>buK<?wb^__sw+WuNlK{7&Cd&<0XJI(zlX`}?*UO$
zl=(PBg#Rf#_$$B)xs2W4=5~a)T+jvQq0o0OcRNmNQ$P?%$KwgQjYqXrI%Vab`^y1t
z*7G`__r)PH+v{aJ7{<x4UH?fOLO<6-H?zviX^<Pi98ceQ%z5UsLr98a!sEo0tA2SO
zA3LCB^0cD{*9IWk&1-i*y63-T7aeosag?p8;!7t%)h>l@4q>ty&y$#89>MZ^_11aV
z*}F6|BAQ9ki}8|KxsP+Sd1iWlyPd|+_W`v?R8QuPZDI4gMi0Y`H#IQhl+{aXn8p9X
z*sWxUWUrpN0T%B*O!BHWMD?rTax`kWsF_bHGU?`nV%HTr0q0l|I&4<6zv<MXN$%(?
zbO#$3EOun~Iwn*Kz*jsl2A}G=4HSOJ^tq_CNX1x<pJ+!{^9*S=fvmNArz2hF$m!BP
zotQ1r2N}5K{X|OuUS!hay1v=+P#TQYzq;mDCj<dFy2QRL?oB%6&2N~qll^U}udjy~
z9UUD9Cu&ZWtuBvbQW_HLRCtFP77jm={KniZ{dO_7#JEkNo17XCS0u#*C12NE#x4nU
z?{2>)|LRVFNrei^bI2`G*CaBsy`Wz1(o0nbw?|g;E7MKGtlmZcrm%^cCk2F8F`^2h
z^Cbvc(&d_~${{Ssp7<}mr@_KA7gTx^e^eN*@_O@(($9X_;?bzcNgh6@<lS_wf)f&T
zKze}>Y5a_?`(1X`i~MZ<Z%=|P%Puxuj<X`Fox<|{eLLVc6Ht(utqJi{pN=Dr-F78`
zM{qkaACVt?M^jo^-?KDs<2qaI?NkFf*bk~s>X7cpK-a_9u;;^Z2iVYU%Db42=Q$we
zx#d+7pDRzJrI{Cm>dKav^MRpl@h@w7j{IBNK_)>CO&B-X72aq!C-tRl*yV3tZYl8^
zs9f~SIr;OxfwjEu!yWtTZ#U!2I9}x1uv@<YIUrY*Cftx?5BCleV<y`s*HvYa`axP-
ztFHbt*F7z5E)O)u%$B5Dy&kGK_#t@P`Z^>eW)qL;xHht;P7F0+(uBXLG-D1UhW1Ae
z)Y{e#P`ZV}nh`S>_Fzt?6HOgN5B-J-!64Qz%#*)3!h?9$=CuFNAn~ZS^_wF*{$o&r
zEnnbedG(n`=W}eZO0`0nvcvrzkALw59swEIT;73fG9RGH7K>!d<=rN7=s0FT(%ZCI
z@$&znbZ;OrSrA*jB-HSfsOBRy_@QCYRDlUL1c-dGK=uwqtr@~${8{tI)5Wt$@T6(`
zdNjz}>KB)?=kXDLv(K5pgN06-dY!)froMOFDO)w8gldsJJDPrvqxi}#(V9yrSR{}h
zlnsLm!U=Y0M<^&I4{e$M#nBEXqjX6Pi6OaWS#ux%*m3zzGc~ckdOO)!(I^xa-DFJc
z>#lbkE~O;c;NqlKfk{blCV=M5dlJlMd6(C1+#zl69`pQG@9m#a|D{Efq1k??Gmn{u
z5bARC?$f(Ee?72r_T$BIN0REZX-$B1A^425hW^}5kvbRc!;ACkD**ABDH4jfI1R@f
z^?$KT$7to)*ds?~JlgHYd9s{Hyn>CeI&icD=Gbnxo^O+X(4C39aOfn#0ON=mT>bXO
z^2&uPA!27p*AKUqGN46ujcS*tlM&UG?%A<jjj>Am)s<tVN*Qu0jgklfq3=!~M2F9^
zV=)6>AG{@rl_EWg?o8<}>o);M;Wp%DW0PJ&#Yk$5ij@ar*JF3OP3dlV2w@1-dP89)
z1;77dx525ZC@|oqa^L3X@muBGjP95J-bEvnd}h^XiCKUga&>3->-~@ddZn&@@G_O(
zrTI~ez-Mhr&#cJ<o9Wu~eo+H#0N$3@GtNVbFV|Kwf7OL6xNglaom-S$ww%F%${`}=
zNG06zNdeTX3kL-S-OUu;mwV?{V#|Naxro7U^S}RU<CPp5u+_c(I6s-q2{toDu9V>C
za=V0IY`XOR^`Wgc;0u^Np>1zu^nu+U>v56Q!G9QBLu@MTdPEMH`35w`q}H|<bc%nV
zDAwt^h?QH}=BU6EYgvsg_M!QY4vud?ciwd2Rjq1=f}lKjne&_SM_Y(XV{f1InqTO^
z(LW>8@3pCWmhtUxrgVLC<RdP~JW5IMd0nzIvO&yVZ#*a5Es#7z3~0;CqgK$D<#UQs
zAdp5atKU6f53v&UqQh6Gnm0nvBg#HiBu&7D*v4``<l=W5i9erK{?`r_*l4i5NvW@s
z)jFmEex-p)xY#f2$F=$Go;;K9Hl+8yMV*UKs9o%ldO8nqyjoi{$5*Jex*c?J8~k0D
zG-@?^VR@IEnKCI%7mhwt>Y=|7(`v^Qu4!!}hZSeRBCXb$PwtpTQdxZ%VyB_StI9n%
zQUn*mYh+sd!KDXeiMLx%tf(yhki6AqT|aE+P&26n-@cTf^bD?0p;|GLN+ZSwE$gj?
z!MhDvLSKnw&k#jyoo-8hZn_;2&y5}@oLmJS+57B;0*$Sc6PMw$Mmx5CRk5t-zD_6w
zLs`B=F?<a>BaA4asDm@Cr>9*6hPR{fd`=C4BI`MXpmWr~8vFHdFYtX5BYa+CZu2M%
z;$wC^{o9|H#1%LTE=`)wYp>G|bZ7ps=F|W7{}n!T;W}GcrpnP=hZ+9KHOHi{@dj7o
z`QUQr>`<!CKj3OK5WwCUv}?-Rb|1A~uR`$eYKHv2Lho_OzNvN6p6@|_7_E;>9_d?f
zRbyxowUxsLgf^=SOe&p9@cRuOTX(_X_|s(9`JiyB1{_yZAm8kby*5j}sTEfbf1+ZG
z++m1+PC>9Q2`cpZ4J8z9vu2<MN9ro8Kjpz%a1z~r`8)@7&^x4m-VyCi>YM^pB3TDF
zk1EIYe-$h;sJ4-~ov!BnS#6QHGOTxag;;B_!ogz36`9oLmrJdb)TF=Qe4bOoJ&z#m
zqt54J<fz}B{_U!Ut<@4d16@=N4%rgv3lt(sd!S2&$BfBAT)uNXvi_>qX@%<P(14N3
z%kTvD%QC+K0RWPGnQ$B`7_Nbnw81CT2o?bVAyfZbP)foXiidQp(tqp71?h=}Y~;N5
zl!*xpHg3NuAR-O{h{Cz?a$aJahzPKC9zpx8gE!n9#6tnT7NbbYWQD)gwIl@}s~k(H
zC)@*zjnsi4B+IyztZbQ#?>Wl^9$ljsR5S2@?=vXKlBm;i)mb5hz@Hd#V7oR%sCED~
zxUyaeRrmkn<u}5=GzZyrG#A47x-xPUda3-1I$pj53w30&BJ}6B;NvdE%I1=l>NqvX
zZqq$E25DbWnte+O_G`bI)>~q-YYIxUW{V>dmMM}aCH9-sIZLR?%IE(Aw}`|@0+3Pd
z533hib;$}u{`bKELkNhjqq=pjDD!uL{2lEH9pG%9-D|mhGR-_(!79J^RohjH<LeTk
zfL%2*|2&AcE_RxYg*XzERz2hNhrXeq8`X2p)pVyB$Q@WJ;^~Si;!xs<_Lyi?W%OUQ
zGKmSnw=+Sju$TPz^yQ$5;_MkX_!+aiqhLnM_ywdbDHW;n%y@S>_MjWOonFDCAhvON
zZ8mfup8hs{W6YQOpmzB0kLa-0eR-T8mSl`s4KX`|5JZH<VEu0dyqxW55bnM5EE;W@
zl{iLUeMrXftI6GMMV`4Bfa-!*r8~@ExESm)x6b*mrD~vgcHAZd1#qZC`pc@Q!DMB(
zDq|&roo++e1Bi=8zqQq;r)5rv#}gy?7TQKPm7gk9HzUN*WV_=qo5o>_Z_hABYH3fy
zP@Z{=9vp?k23=eHkM>%GC%3y2Y`L|+J!*3GKT9O+&w;1~ibS83v-zmgOg?n?&^^(S
za&~rB)cz;_B0xxdF-N${ejQ#p*}H;P^rGkb`v6#x!Xm(wYayx>=705|F0pb2k;BJ&
zg1)lL>$6Sex04_j_fhqP1u$dPj{P|a<9BGkG4mVpXtYuKRdeVH{ilR~oCr&y(J<SW
zHpJeVk9Os!hoyIN7%!`6jbR!EeHudX@429Jul`nOeQ+_lSg2|J?_6bXwxfSE#UFaE
z9_~kRbhrpr*MK_B2yFycJ9zZgBeZhdF)zP(EqV>?1(mp@ms?7qy=ZLPpI$+s@LR5I
zz}hc#1zj=wa-0xkbL14Sh@XY7A~biRomYO#!{2!R=4shcnN1Mar-yC-NL>mqv9Sof
z`@E9r+COu%L`N*e4utdr@`j*0>-n{+`G8|NB!s3PG{89_zXMLnm>T5MW|`<aX@zGP
z`+6|{t_F~Ozp!$q>#uDRUpHqvj)(E(`g_ZS#Wz3q6k$0`x;7QTc`%wDfyWsjwX((G
zSzIgoI$H5KM0gwg)u4mUCx*Ap?+h?=lgF1Q#-ay-8>$~t$_XBwkST(@cvtKfITXh=
zTxeYnxYo2ISjNkTeK3kBkZY|0_ooA`ADXy(i;USFR6~1w;cvrnDiaY$;YkpRcisVe
z5Y?l%xzyi=Yi!$o4;4*(-k9`+Ns#k9B;fD?UEgQ33knweSGQt#yW758`QaGqWPt`I
zFi+LpY2CLehu7AxbM5267R~gGIECJxp`o(e?8ljlnpc`?V=e1<U0LLD_?q3rXpM5~
z$+1S8_^=|$m=Y!Op_N4gsYuig=aVQsH(jMR<wYs$<%$~1(;Ad5RMH_7XNpb0YLn}p
zWmkFrreuThV7cvTqiT<@7Xl(;(b;mHin1~~B{j9mhsZNch{wgsn!3B+staL8;9IwY
z#uU{5f(VGdzc(_ZV8JHMTcRLlq4IPKEB6U_^Mv53BBN*NImG-e><|y&pW-d|gbLDK
zDyx+u-+Mi~)+#bcpnq|9)2aQQjQxu!AAIBi$hlvCT3qt<6?SAjz&*t2p;iB;p<D48
zWQx)&iexPO@gq`tBHtMOw&bx^sc5#mVPLCsD6WviSR4kie5Ld5vkR74D=o~<gZ}4q
z(*<MMgx96F=VcD}Pos1q&9~1#CF%7kKZ388aJVNMe9hh`;k}IC_@hfdTQo4|ZvCh}
z+Jslg8L)s+M(rJ<C@W7f={nb~;=b!et_lOr_f|XdAB=bjIwwiENFdEeW-J;_A#;ZD
zHp^kyt;5sAzUfoE%FVh8&@LyM$Fp^K3Fpy<c!xU(56Q>rx13Y~{ZK<mAMiT~z?;bW
zzUGKa&aW0->RB=G`=xV_#3c3S;``-D(k-tPDDF2!2a~S@NSZFyxq!p%rs907G{E0N
zwwC`LK@&%!4*{-E8Q)6ylL16(IETcYY_Hab-|Zo;Vf4Jfbm(~uC%fol>Cuh`64T6#
zk(UjRp^$j;AOZ`z_5Br2^Z|JD-SXHT&}~dPalZ?X4Ty}Tl6560p)nEbX(x9%ArScl
z7<o7-4IabiI$n3SpxT3wv{u>Pz1+-dewG;2{)PJzROO*;wR}yHt>xh2jo{R(Vs+RL
zZGD`b&o@Aq-x4Za+mdkP{~dx?Xa{(YDa~cq5E>@%m@iW}5<R*wMop~d+O#ERLHiF)
zyK>rv-^{WhsYFJ~Ia$wuq{bpnQd&?!Lqo%Qr4iyDcZq!c0I4iKw?rln%eC}!vNnFh
zm+(N<NxmM3ASo${hKeej=fRp<O{8mI7MRW{F>w6uu`fWUTKp$wGI|7whzt=wg03l@
z`q<OkP@kW{*J!uWur7G<uN_+C!UjF@^Y2NkB~+^RC85k$LulKWFH}7V<rF+#+=D||
zzg?^1=Pgw<$=E9s<4-jjro=2ph5zb7WDcQPBcP!+*SM^dLmTKkZGe3(`shTOw|CNQ
zzhl>V{<QZP`x=-*3#G)~i*u;Lg!<pwsY?3)I`+2Z)9Y{yi;F^aI!m)na*F4#QY)n+
za~2hUyfJ_E#;?RlchQArX}#o06q^MG?!rO(q2x#xzADa@YcY<i{2G7Ufh%u{iwVRa
z50p1r4~t)#e0{i$ijQPcO}KdCrCm^GFj#edtig*XN^1*ed@~(@DgFo(znH4&WIepd
z9HF_X)JgstdqKZ{hBrq%GMy}`XbOZB9dpV2<jCMg$ChG@K}m}4n+u@iNn?theqitX
z_p$k^+FsfDhD0JzS6T-kk4<!IVSr@O=R)XT6oW(~CP|~}kaK2NIyRB$8%6Cq3b6~`
zp<1zws1e{|j{J>W6`HTU9R6G(X4Ld}c5*gV_=u)*ul~lU-Qvi9h(p1tK;d*{qa82N
z&5?s@u>76doh#`rYmu`QAzcj~!o}BlQVF79?5Z(TF`J3e44cE$u+w#yHSEnN<K|#g
zba7{iL>;1X;cjB@ubslS?(z?`!02H9>Z!~qqTfv42tF1WCB+$+8!L*AykowQ|3QGP
z->`>QWTU0XphNh+0*5Yd3nyRPSNdF09zvDYSNk%LW^v-Zfn?QRqx?5ysE3_QCtyA2
zPybPIImWT&UGcFC-Oyt*YQ1E>E+bYf&tP<7e17ZOpFI%1M=0~9IFH-;H#_}yJ#!<`
zH+|G?1foOvmm1s>R2%?3v~j2pu%O)4rd*HKyqFb^`$U}rMUBw`cbT24T|xkrX$p=D
z67tGt6EDy75n7`kJ$sIW7-p7n=T|KvNpYg2!rUy>u4-ire#^y1Fx~INW3OJOB%@+E
z>31KVy_n`BCuNAkWur|8*_VT?pXg8i`}6fB`762<bCvor<P^m#O=g0mY0hC|SkSxo
zDe7BuhccP-Sj#Og(DfpZm+-^XZ3q)S1L3Bv#!T2+lO{=3e<rqL0+UI;0Nd0w9qrJB
z`w8GKrH%8XHt~*{g!%OL*&=%FJV%^(aNVOSBgLqN|N4xw)yvSo=b@zhaE-nGt9Dwr
z8QAwQMjL?tJFr_^o#mHZaulg*W>Q#9>PpV<bjCJzUGxj2r*{ep3gy=a5G@mvLI?l%
zGbWQHW!+IaIFeCdZ=!Ulg9Tb>%Lc7T{=HR_!TnW|oOAA>25{dH8X7X=2^EeYbQTQ;
zJ8BasGFzZ8(5Xu74=C{Yd+@cuJNC@<%prPG4Aj|Dx(F2X_}2zuTCRqTA5TKU!T=EN
zQx|AIiUAlFK&r~x+7G}6|1!&dwWU&2RD8KDgHbKkM6=orE5Z5U%PyK4L{cOWZ9;<T
z^&@O&@6guyRhTZfEk~|LZK0&(6mVo)e(U)i0#<Zq;83-5+|`63bcI&8BnMHgH6cE&
z;l<F3>vX{jmv(n_zsy)vlOLD$8-AEG`jd(=uGeH?ESsgOK?qQ|#DPn8S~XQ=-Itjj
za_T5K+buNQK)Ae7rA%RT+9BYRgj1MIz+vQq-1T<NoKB_Sn;H&1su^8MDUxjK2-_UH
zUnTw)M0ztkq-Jh3YW8G~8n7_A#Gy9J=cjXCNiLQ&5f1rID4ZcAA9n5mCOIM&K3!#s
zTSLXVh_W#{AqqcfdO*X*db6I1^tXCd+w$h~l^FHU?mR%<rZ|Kj&w(cB`tPu_i4wKD
z+6Wr8>~^z8zOKuV-e?iz?tGUY2JjN^qidFo6vlTHt;4p)6Rk?G7u)R&$8csmgS*2~
zZypVfKmDBLl|c@das#^_4e2Cc5q(&ZyBHgN1DSkh2aW)H05v)ECtTV-nXxU###?`P
zM6=JlK-N8oufzG8?w(*F74;P$0kW3B_=0M@&`W?p&slzWXR8pJOXXm|tj{wAUrcau
zNR_`qU^wH0Sc?FsvQ!|6->!v@rL3ZIV)|tDvj6Hew<zpDuG@DCx6wwep0a2AlUm!Q
z{<yYeBvjb?ew0Fkk)oYYqA&cvRq|MDrxHrgwRy>lCFEUV=Hl}Fc-zKC+&)sp6EI@A
zCtB0{dd7J?8?0(*vUV}~TEgt$ZWk6I_N#lyt*LWzxh6D_99)NxNUYbS*J6!B**|=}
zu3sg~S+QS5NXUk`)lcpUqQr!$!yIb0ffoqYq@y4fNDi@`5wGLyFAb4Qx=)rZ4@wRv
zGRx$6oc6lJMx)+((Cc-XX=x8NzZH3PL9tvT>Wel~oNM*A(e(%@Bu{eIV_`pksFq8S
zHo>=;XSahg{G4Nb_Ex?<4^h{B*NBu|hlc2i&}~2T&7N$8x_!G>wcblDjq1cp-Z=kt
zbQTjBLK4{A?bJH2@-URQCD~x{U}34>CdN&7sGm*e#XjC<J{2_QA2QXi`o|I8C5yG3
ziKC$9YN~3?v}|>EB#|8PXgWV2wCy2{1mK~ktJ_nf*|2w0p;AWGd%s#GXq~2ZwxVKM
zX}tzWjA{E*{GqlMhl)G(mhBS$69z@cpSMV{mht+aXHs%!E*7&40k;$d&GcWFC%Ibc
z<glFceZk~oO}LP=GlE`Zr-B+keZ|~y>WU{F*f^Gorp{)O_WcJ09b_<m<BQE9+kU54
z?_;@ioiy%oDG?25*`9rXjsgceM$@SOioxc|ybf%7GXjPpNJ~pIBUXVtmF(-DVzRhh
zsCsJ1q!#f5?y2maED*t`4wVOZ;+;0x<>HcMIvFl2>ht8YVwAK9@rLP`?}2+mB~m;A
zri}bugpy|bXG%5LWMW22$Fd(zKemvi>?caNvRWPWh7{||SAxC;&o$ZDNHek8e|K_J
zf!5NH8daRfQ33~#SPr;!n<6UaQ;j*GrZ1j-mN+Kj{%e63HNz$EU@89>!%bf<@;tXt
zX2BaW=0bXl_GGuD?EU`PQ`G!gV9pM3JdyA|E;4|lz}cgAM^gOU)wA9j;1>Nk_`8JA
z4)^+AiB5kg<U%*LFhJl;L2~JnLi!>(qLr5Nq=KTnO=D<2U03%eDwcZGRYfk(Yrsj%
zNnS-c34LI|J4YsEn!p%tnRqk0Ok_|-sRvSAtTS_ZRzev_-r;tbaz0&Cm~P&r%(=k~
zx)7~>p4)e&d%>d9&axlOOL`d`=tM}e*?I`N-XDHNhz4aFQ+p+lv>_@hD{D{W@xSU)
zg;l%GwR^y7v--dgt~mYHcD0qM-&?A^9JJD^3GMB%caB!96(C^n5)}W=R;$;oa6xnm
z{sF%)1<{YVA=9p<Y?s}P)LIig{S9{*Fjc%6y_J4TsXnIw(vd5`_b-IFtSv%6tXvfn
zly$@NK^(%r)mxlsJQ(_&JFL>1k!coFqw_R7b^mjq*iM&eug*V1(gmlsM5SQdG+Xck
z@VipIf=2qH|L^e0a|7|*5__*SS-15jla~8Hy?$k~ZbI>OU4R041Ptrw<c%w%i_z!}
z<K5EhQ62jJp~X`)@vm2o*nf}2+eMv^L^BK17Rtw4%qlBAVX93X5oTWvi_~{31ECRC
z;eS((Fl~L1zNYz#;gEZnCT#p^O&f=IxxMVayHdzu2;t#3$dr8Q!6BBX??Fg<6cRJy
z{&leS{7d`V4HYKlIm|cIOyl+KEzzy{eR`R8_nKd8u-l#f^rsd4p_5xlO%v73YC`Ib
zw-V>IF`pArYSn3aLB5D{fkDul-Jo)=6%*nQ3v$HBNmKl=>6(0!RzWH%D)nq$j|)As
zZ-6I>0T^3L9Tq?MRJvmcG%~r>ogeSE1yL&ZqXO-5w_G?=T4MR~&rwqY#GJAxE#ph`
zt@$59B<-lg+iPt1*(f7*iMMSZ1DguTN-iuBmsqzRVR*3>kHv$|$bv48xOSP(Nt)fS
z;T<u#X+f0Vpjmh<@lXwadEV@BU@@vGX$IBJ%}2Dd#t_JeFUWfO8Z1;Xt`EDF=%ekv
zg!e*Z9cPTsVt*qXE$sDKX8zS~?LX%TNsb`4-d6C&ILB7XhdqQYMYP;C&9KmuLZg0d
zzol5hIym{$F+<$BM13#6;_~af<lgPotO?%J8*j7Ky8qh8CF$u}MHmCR*G(4%evg~A
zDXOxjtx+bURwGTW0KRA{eQKG31ykao5ox%%=WKqyQa>b&X*=b3>l}9?0ClMy;c#r`
zQy!0WlsOIyqCnVygtZ<>SID4)!>F7AjL%Pe#Ik8n{<=LAu&1*vbyMxNnHev*be$A@
zZLK|0r`Icf`=Ouykt0S`_R-BNeO6Az<RjiCjw)xkveJu;bg`Wo*eW3Wj3V%y>!0(`
zI?F8HvG!GI$iV_MN6h--u-FCh%tj<c?8}pjJ^8Gw!5-1=F8#!KlH?~Vn6$JQ;ZZ9S
ztW%;t9AljE%bFb|%NU9X<?D9bvzc%cHEOf-ia!YbkinsVFRud)Px!X0zSikVSHL$y
z#Q;O!dyzb=pa7czC+w&Mn|yhw(s{BI#eDGio}Et74`Mr&tUTJzRYySYzO5T=6l|{e
zx6JU*?xuWRUEwO@AOG0*qsD}!9Iy#JT;(Z9&l0&bcQAP40qfv_mT4t~<bsKei6OT;
zNhXa>xMYkc9|nQT!UD^W)RPMfr^+=U`<m10Fp1rj^b`^djt6zzAtacJZ}~n<Cngj%
z`|T@uGaMlr^mZ(em*Zn`82ZoJ{+%(AeLYx1$IyAG+37V}q<V2aZ5^u^-CpMe6}M7#
ziOTD+7sU5IClLMa{p|Jq87qPGUAD^oO;oMYD{BM5uLeCE(kBi<Z$J6cl`AaNosB}$
zioi~pDW3Vt$vpndD?mk17RryiNj;Hx`_?|w(rou2c4Pk|VT#~Matn5r;OT=(_*eei
z4Kw87DXzVtTiv8fzVdsViB!e%B}ffgH0;Q$fi;g!@$J3uzRE(h*xwRkK>dd?+xY`)
z3m%&q9`Djyg8esPVv-$U2gYO``z$nuW54fWET8oBiJZj6@JZ2grO{rPT2%%KSx547
z={%QFzbHr=@3T<S@;USjiW?UpEguZ3%xXqe25zigk1M1*apnZ~zMEp^TlaLEWfl}|
zl}TUvOV;lNLkrI!PZ`~VW?TsLFfb^fp}&Im{l)8Q*-lI1#S_k44a9ZNUcof$A+Z=E
zPi)eo@JKVvOYk4&?EJ+trOxW6oU_IOn>eHcENC=lvidlYKwe#G%>JKYsf5hP{`Xy0
ztCgUeOu7q78E(C8eSg2hCO=9~RHTz_H4U_ox-iL{RAv#;udbscZROwQ@Mn5(`(GS>
zD}42^goU{jobBF!SbHWnhFTN&aiW)`z7#myYdQx|ir1)+DAJ!pf=G^XcukxW`J$}V
z<V0eW6l(TZ>ZsC}&0YJK?0j*7C7OI|>4(vx^^%5yX9hz^0KcT#nMdh3wTG<Uj*7eQ
ztL+aXob>mg@AP}ycEJmZ+yNPrT&b^5{x467BQk*4qt6>$JuWQan$EAYJ5CP6@cHXr
z4`Pm3x#MaKvV$_1C9iw&KhcG`iUM5^C`jryb?(!kaZF)uM7C$|mM36v7t<JZg_Ehf
zWbI2f%K`NZcG{h(FCMre@bK`sW{x{F0_{r!6c|E#18IKyFo(A&J*Cq9$t{blCvR2p
zJ*gMSKe69rK)kIbeGvR7(`HR@wWxxA&sEZPYbGU<@Fy;ON3v4V!q)6C^oCKL%W>_=
z6iOiOYi299K>XE?{BY^BD)Zdyd6RPc_1(EI&UzE~KhSi{y8D(tOVdBwWaJ^Y6R`Cr
zdAJFwquTxN;;9d|(I(z&y%0y9i9#QrrrH}%Nsa8{;*qHn%o0|pfk|B!*C{dPM$h+W
zd~tuEK9+<o_~+gq8H<u2U@JA9HEOoSf27ySD7V|*b4atSwb)bCRMyXYaKDpFv%TpE
zN+PD&{`tXNtP^5L3f=I07Xq)UQH%-g6>O=qzM}%a4AI8Q@Nw&6C7zQ$<z@B^4p^Hh
zGng)g5YGJ<T~Q}hMLuP-TuH~+LxYLup7db*A*vl-FnrmRZs8;zSUWqsoOQ0qkm}SI
zU>LScNfHWW@Fdu9{#jc;;XX>kYWvFrG90AMo-_@~Ow#NB2)G-RAmuaY+Nj>{N`HBL
zo(|5K4IF}aGZplv)&zv$V7MGeJ<)3ErC&1oi=J_S3Px0pi%V8>{Ld?xHn3c!19po_
z<S;uf_K}E_va&wN{*k3cVg?!1(iS$CXmPTOS99}YFof+Q8QkL8do5T|3uZ0v0{_s)
zqZ>}nfIeI=ALJ1P+oGos&+OdtUlx3xJdJQ`{%CD7#XD@bkuxMKmJ6F0Xm^?>_g4gf
zoLRVOQVE}-rTyF{vgZ(B$E(SL-hGsR4F@e0o0(Y6=JJ{d_&puICXM`==rejR*XZnN
zwmGvuU8`zRmT$Q{ug-np5jtY=tH?{`u#JqYdAZoAE^A&xA^xe_vr?n=u<cI-gCp9j
zD4tguJ)2JRwD=>F;pOVuZnc)%ZoOW#2O0~NzCVshF+Wk)j9^8jK-Q3=&G*K@s<@m<
zeT2kcZ)mm`ach)?VMp7}XcXqT2Ha&8&h>??jN_QpDgez}weQ#5+>)I>t%WbS5=Cvo
z%A36r8*l0b;X8hvM=~})e8a~*pUm6F`r8ug3={AE2^xp6T^OLv24V&&u=H4t>0wFL
z(xvG>9R*tgPho>0dbQdR`)(d*J58)^0U>j9u(B0^KF}92xKPYm_$L}{(GmuZe6{;(
z8pUjQ*&kEvX3wXfo4aIMW@!y*-=c5TjZud1gv4!fffE9YjgWr`e1A7o#8MIlO|GMi
z;9$!KNAr5B|Al=-Rex4%Bw;A=gb3n=hvov-X%qL0ucihBWDnj$EXya-H=OuuQAXiR
zDicl<IG?$91X!j)y8Dxgzd<Zgo*XDH6aKAD#Zmtv$Kkm?gzu@V3YUwbN6wiaWSNF0
z(B}Cn@sCB*y$HEx#8R_?XkeHid0bT~V$RQ2>5IcvMzbT5!$0arhT>j9OIf4rQnS#X
z^m?lic(4kEyvv0LUpf9I=JHM1HYX46G3AxXf4)i6AHU0pM`4#0FuxPGTyl?)!ZCeC
z^*W$<=C~_R&&`hZDhtfu^f2f2E$iy>IQFL_Q~zXxXZotWS%r5`v=VLDAEx@?T80&^
zA*&qXqVc2GcB7RFv7bFPr%;|98uupdLQyf<#@ff)#5jg1?E>Gw(UJUVC;D;1U7pwJ
zyqF-%VE?ODI9#u8mykGoNO+Ny`z`NXz?1ETiZ2+Q<4yU&P@VREub_gk{<a+zS=tY2
z54*OP#YsQh@5?XMR@UG0rPhWT=K1}2{=%7Eq0@iu=ilECZj&xJ>D6E1+q5kabdOFW
zvBbo8ma7=0)EsKdMVt+A(2xtsmq$=Ia%j`oo}7UEUt~9#<KU7Gq;!nrx3xe%LwPp#
zuJe2nylclp0MNwYE>Yn#F3$@;x9$_C2DU(32ecUH&4QPg%@VUpF&42s5|IFZaTT1d
zj4fq{zslG+qB;tJ{KZy}IN^KM?8b?pva!ff>k~E4NdAv~C?zler)MkDCLdkqj_R+~
zeX#Dn?DW&@XH6Khn(&Wj!;sak(b&*<yZe}<26!Y4hk*ydf2}z5QHN#gxQ{}PyuDBK
zo7m7E{jZEA0ZeZJXC8Ruf>*n1SL-xdmv6I3suk)}PQF;tZdoH@?hj9BFURqhC)-Hl
zBSjK+-u`>93vG5diE^a3;|}4h?w((&3=U>h7XDcASLnqD>4|-BsB%63_3~AiS9R4i
zl?Oa=%F5>)>SkQ$@u7<v6ZCmru<!Dow03=P<&l2Ad#x~r=h5n5Z@xwR<6)31vMWi}
zTm}Bdcy5V`0jwi8x7#_x`}L-Q_h>B@5f^}p3UK<`>LqA)-Le#6<^Gr&Yd#od#%`HX
zmtjBC=0jbj#Mt>@>&0L8Oz6$;Wj;E1r2icfx%w%me)n6N(s^#*k>SLChRbdTDq)W}
z9=)M%pc<CjMz_upmsoSF-;~%18Heeq!VQQ;xr56PBTFR9L0^EBvndxt0#yZiX}7p6
z{y^Z{fL5<Nsj9=zNa>^GjcPUnht_bMKsTJ5@z&%LW(tO$9=FEr80dV30cjZ8uaR?<
z7GfR3AbArA!mLPn8`+Zli`Qhy`I|-FVnLQS7Ogw-FCOUfN(_exKNh;m2m~=PIvN|k
z6{BNo*^;e$Tz#)uZ<FlEtJZlTceCDBtuSu6%z3kS>prIu*og}M-pW+?ExHBpN4FnR
zYb(Y8<FMI6vwq%H#pOuCm^FW_$tG*r-A;=+?eSuc*+8)bSI}QrI3%x3T1BAb4F<tn
z+9!Gx>W6JZh<bW6-rEm$EPNa`82?5Se=gyxNg#e(BQLgDz0-LZGm5m0eWV<%Z}0z~
z1#q)?5L$6Q6P9XNo`I@_;@?8jHKw4m5H@{Teo6KZ07n{bMkm)7)yIsOX7c2mijJ*E
zIaS2;{_A)vWZ3n5$3KdzIReZPO3JT$l+m&Kr5wq<V8fHjU^8ge7rUDkbFix=p(p=~
zbQGSUmdZX=FQwS*busY&)gvBJ+H5z;bPddqI}GcsCF2SxZk)Eyf|Ct>>avyv3%FLd
zQFgeKdtLIw>bS>TLRls$(rErmc^7J|sTv4d;6rz%Mg{4m@HD_~UFl4{=A{2U0{W)$
zmsly+IXUnkm4T`1?sPGFERBv?w29X4;0{ToGIL73^L(a2)M}~KrG@#7zxTnG9iBEv
z^f62cJfHm6A5z_0#vJg$rlK>LSEF#$+=ZO4uGvFsHkL}7Afe0aO%vm8|D3K5hbRB@
z<0DqPdGz@*kJj%mYo)|?=GPh^yyFaZD0;|-C%{1Rwa26V4o4Pr_BBrf>u|zc|07S!
zU?laq`?1gH?g%!2{F#u4c1f-+<KEk@V1qCGH$qfj>r(4GoLAKUVd^adqWZqDQ9xqo
zkP@UzT3Wh636UPUJ0+z%q`SMjW9X3X?gr_SZtmgtzxTcG$N4blth4vpE1&gj*>;Nn
zbfz#I3=B<t7CUt+QEv3i2J(@ean=>ehb{rsI^;wyu+vY}?BVmS3{=HDwah^W<udH<
zBp9FX;cjhuQi>9@z}>4Sv*qDeU2wmln!J@UU{w||3d(6U5jS{|LpNgYo|CP)yo~?t
z&pDHiK6{uKxCI+MjuBEEENAXblU%9{x-GcFs)BHzL8dK1pEyzwb%gIY{n*XlulbQT
z(1`SQ!5K~Qu1`Oyy~gI@VRD>zT^#9S;*O+H;(3T*7>R0NtA4l*QDdCu<$B5}zWWm-
zw*z5k4uO<x!7*rG#_;t?e2c<hET$c+mqqz9pAr4<wX#M<D@|pN1F-;!XydZ+*BZ)F
zjVGM!vs3e@B_>5jnUCDErlhe**)wGzX(1=INnJAUzsi_t^f_8`%jozXW^>Ain}pJa
zOEKx5w`i|okMB>1x6sel^XVr|>PwBCwUzxbhEF4>L^0Km3!HkXycp)&4bNTH++7f(
z-(L-emvZ$K)auvSNPaEISqN@$yA<dyV6TJw>G(66aE28}X;X|Wn2tSuvZvl+zQ~T_
zHtZ4t`Ep^C;P~wWTukr9-_!IKvkBy9*1i&@m71Tt%H^BitjT+pPQ7lYq$?wptq0G<
z?GqJN>cNL=m9>1Y&xs${5ByYF;|C95Y2$<&90yxJAe*i>JJxLv7;81ww!tfCD&U)E
zfVR~7D>*rA*8STbciqwfVsob@E<*Ma8hzLCi32&LBgb6h7l*wYq6hb(`K!@7<~%|3
zBeZf53C7MfJ=Y;sQ^A!7C8bP!dwsvfY17TF=;+Ob`7zr>tF`t*4Ij(eU-we829u$m
z3sjh#8_8i_6<h(wtF$+CdE;l&uPIdSA*zG-ZL<9pQY`o3+^%<fk47@~U~#=)6Du81
zqJFy`_zZFP1UuMsO8zi@rWpprUZ=6E2ZY|@E#lJ3G4N<{@4A#xh#(O!>ZlJuK#ZZk
zAO=r?AXY1zW<kI>F%}i+{M8S!JJsxPPPXi$c?(-sJApp#V1Ne93O4p|W`oH2U>K6l
zw*%Z2f|z|e*qu~4o!FP=6vwLLa9L?mr90i+Bz>26KY2WDe^Zi#2aphAeeYaX{a4=;
zSew|77UP*+Rbi&?%dsr%wX~PQ^Qj*{QNLKJ;%|dZ6xebh<;%TAvW&0`uB~X-Fo>b^
z*Uhq1z=U%LLlzeeS|Y|K6?_GnR356uW<gJ9cw0$om<W20)*P-DuKEdUUs`A|&iioM
z%F;4cM15NNte>)*_Ud7WGq;5aR=m;9s}DPh<C!5icoB~+?^)Nb9P76k=7Bky$?c;0
z8OKgg$-dsmdJ-SYONMxOvG+py2QP01ADvcGNGWbvE}k|qU5w4n;1rxDZg)?}K-Q#3
zEtS3+-Qr-twCHz5Ew4-ztFa9J)C0clz%N2gy~v4Ef}x4i{W@Nx9#%iS{>IGK*^CRg
z-qvy%*}i_;Ue6&|n>D@i$E@LT8qegmSLqW`25}5rl^9tl{jQ4Y56HW&|20@P$5|}I
z`P!)pX?9jNr&i!1yvWHdO_mrX7y@BswmypE%8>t&n@AM&66p=aD|~K2pGq^kWYvSY
zMYG@fvs7cF1>SKe?_~3X?+!E@PB(G$8an!HYMNMo&Aod&;pf&9;!k+bX(~3l>@dLI
zA=;4xcWJ0z{Rb&!w<PHXzorGfrW8YaW>TQP{&-1TP1YcwfJO8Y+#+k03+Eu3reg&S
zJxA4xWRy<g7(aZFqDF})rbHs#=`qTRL>4&g)36Nv6+1YdeCZ{t>WU5qrbLV6Z~akL
z>kY?=eZK6djchtcCGU3ijU9hqFn;f^-@<vwV*Eua%s;FT{lu{oR>SE%$G}ssdzF<_
zxu1iBaXN7cC)_>Er&22mVzn*w0)Ig;AcLeZ{sqTNgmjMfb$dvO{^^@6F%#wylH5$>
z)Ivtw(m?ZX-H7i9d01Ou31Z0UNW$0mHOE3HSPX=9#Oztx3}RxpnwUwf2vzpGmgX7;
zH{ay9{aflpP||46ybq@|6dburS#!C5centdcL{+MI9sn`dhY$c>fbuB`$l~2nem(+
z)?M{$P2u2=+qdJA+me%b;q$s>&fAXyO%Ebg2Zt1Q<<ao58XEfg!|jfn#zP$A=EOBi
zjzsXNh6;X=P}I;HriWeUeVwliKLkcvasI3Y<)?VUT`ni>lyG8M4;X_w^~<H>C<&`V
z{3ghS5g0Chee-ctRXo?XyPttiHhm}kyp-TDT9d#h(IMQnB#rk0e2v2&EsM(H5M9?Z
znP(gQs`<|0K>JHaMd_F`s$sip`9t??4!)H(=QwO=g@-Oo<|<pXJo{mNnhsR-gY!Wq
zy;)|k%rsqAlgB3;U))8;`$Sq*^#=G%DU79s;h)%Uv?|TEy3VZyo!t;#cS15tZ1*4X
z2$=YFy)&o=OD9RL)<xHIDM_Z0y$K!`NBUzfxdksZt^+TBSOqGKHG*!4aKDKUl_s!)
zYpv=duSd`JE5AbHH4D?4JD*^j=&f<70tNh(W$yV#ewaXYPI;!nnyPLJvD9)n>(9Z*
z6Xd#il&z$z!I8-1FJ7?{OX~cykIa_cl-^H!QYBhe0VyG)&;<hlRpC=|422J0AKhr}
z3K#B--ATm#Io`pj!a*3k_nkl=EpK*fuT;u!&wf`a?vSyv9#V)^T@1~1Oeg&K@b6+*
zYkb&Ms$}8)RssR}r$j|H2n^|TGDpu;_JAEh;qD!S&u_-ZxsRz${l6v=_t&E5o@yfk
z!--=s>!BBC&RN-iUclP7RHZ%yWR7|`<pQh#@_&N=P|k;-hqVlSfClTu{~F*<9C=}S
zvO_;hVP>o8vccbZ9FC-!8t!^mYnd_Jj?>?ySQlkW%6`Ep=zes@`kY%KpCTeOIR=Lb
ze>~IPyNCtuSy<(x0N+?hk)#-ElL5U~Ax%w6Z(D3cbnfraJ}0OOjXyR;X3hfE>Kwki
z9<E<3`d_jYRcit;%bB^k6-}txRg`}x#Wq7f?EG5L3%!W}mJSkmYTbL1$3xSW$!Se-
zBs_M{2+7Q^|J>9^2Ag5Ydf7cJ9pfh*QTs6Qzg^7=)(At`RBQ|tNK&57#~xGW!?}%P
zEKt-n=W;-#Qt$%1NKd|clcT5#O-jT_9Cld?sK~;;2t6I*Ob%9ChzhP}t_}rS4Ml!h
z)dQM=$A({bvznrzZ*jU{k-G|Dqt-p71%|LCRcTEIxHw^&TJ&;W<%AsE(7k?hGNn^6
z^T14ox44H-O--?ilib;eHAhM%(u-^P@=P6e%uk;u!XMQ`9a#rd68L!+3pDjzMK2G-
zU+q&e-0t3ws9*}L3^y_)&}K!F@Cv*1et&HgNTOtH;SSKZXFncJ{A@M&;`8-@80ayk
z|LIM$AsB39ULGkH0G+-VWe#-E7hGi*+uma(?|F>61roCja$wUk^@A430NifVkAK}R
z`!5F9FfhkY7Bwj36)25TZ4onWQ&`DOma~oT40qX9-gTYaOfiV0X~{0ueuX_+=kZ%Y
zdk({BydCi|+L<lO{QBi*=vZLc41@W^&)yg&C82(Je^#6oTHA#$>Mgc`pzQ>)`a=%Z
zB7O}VBwpJvg5a?5J}F?>_LIOVe7QCxBQT68>LifEVA-+Z!m`&fjb^D>1EITL)8~wl
zp?QqqZ|@Vrx)K#xM79B3SK61oKn64e_%MQ@rYVW<63CRd!eREKw)#^1e+d(`D8qz6
zEOqrBr<5Vw4dP8!bB(OlwFH>)$vY{br1K^Kgi7f8q5_*Zy(|QGzzdCaNu4qrN_dqD
z#C(zZr-rcFAmS;I3ZEALmMJ-<w-HZTjTG{0TGHcd7DzYZOW3k;Dy|yp_VD*8d>Aq3
zS)J+qgIGzW4zuOskfp1K*nT+>A}P;@{gSM5%g_kfgTR5gMx$A$!zE^+#|Etoicmb5
zi*;<0Mg?nDu?4rx*7QxmVmY&!S8Xw!ss`{B&AdX=$A$OgQwtK4$X$yzrgdWLok-ck
z8@PcZ4gr}|rByd<_BSs8I3hbXady=`P{s`Cz72g=*Nr&k_$lD>;DnK|Aa^hjn8~UO
zr-$Zhq1MNZR;7MBZnA2q=>)7(Acs3l-;@4@X5}JuUs+($bO=F;Eg^tC(#3*N9xlFJ
z$Ddnt`qS<%A*GYQZ({Qoq@;bXCK7y4uFAuD=)F&_7*TMHV~v=^5RbwK39?s&t3Z07
z=EYde!lRz}4MbxSb*lwVMw6}pi)=xPz-%$;7IgBb)klOMPqV44MrFTc!Sw{7!>4$A
zLv~n7rp{_CZs;C7OdO0$70f*IUPtk`Fo1p~^60Lec`IiN9}%;zH)cnvg;{ZYwhz|z
ziNhY+<9f0YD>dy(p~a@>r0r986IqYOaKotCVmA)A<M{1}P?l8FvCfsO6{o;%k}Y>B
zV=Q;4EwuFUV$_uh$XBCOO~>s1tsqTzv`Js9M(5Gx+p-(fby2rRe`(ka0!}(P5@p&C
zQNH04Hsgk!l7H+F)Hl#Yr%|@Qts11zcNm7D0eQ`WR`H6woI-)8I|f-eW$opxBK8yk
zKqX9W)~`<ktrzyE+UF0sw?)edOqmOe9yR9@HWf;C_Ok~iS*sNy4tkRT3#sB<B8H%2
zPSRSyz2IM^Q;4c&>hYWXD>G6m5czOXrmd!f)6)7no!Bw-3sUvam58--+al)`M;IjJ
z`cq5IS7M+_wZHWnvn&BU;=k9m=QQ;O3*ml|$Vq9Nu-uz;whqH$7F~_Ios<y79F&=g
zBiX7bk|momXHOpWf*X&Ao)6}8CBH560!W)O<?iVurE3VrsT&*Qh3^Wwjp)hb`5W0Q
z3xt(^80PUcnLzjcc4$daSXz~APSE0O_v1UP!|`v-T+zust`A-i&@ldVX(g!eQZA%r
zANC=|V;lVQ&vd)>P?IY)siNy1&;OupFB~&2I^q1anHx1wz8ZxmRWhfbAQdms^G0zr
zI}DAyB==vuOtCKm<)`yTX)BhvB=0j0GH7W~O9=%kM8rdkzz(-oYF1LtxW(_d0-oS5
z6{#YTj2j~vH2D!6^BqIE5OaRbAKzF`3LT*_9suFAVq}&z`GL6W#tX9iBzc1w9065L
z3MwI+Qc3(ByN~wA3#@P#m%J60$k6cKF&B*H`E|E=-ETA4k~f1K2(iagSLXV$)78Zu
z+qXdz)*&k$uR`2AHu0F9Au1d=s>fNxyO$j6=n1%*9b?<9W-G6&^>%XNKpPn(=KRMd
zcYmbobp=Prv|4pL@3VKOxUf~R_GZU=e3KPmd+~%T#z70vFy|Tpx$R9bx9{@6PtIMu
z?0eVES9jX2mRP)wrJVZ=&yBl45A@~NY-&Zipa>&^XH>Cto|WZ_Ma<PS3>l}Sxr#%6
zjl?N~O0e}x{-q(a7#H~!<*Y@Xc9`ldU3`(%d;%-S=NRcPDY|Npq3G>7xk25m?q53Y
z?iGJ(21kwI;tCH?==EC7jAUZqN@0ST$SRn2U+3aE>pQGAZ*<Y9mmUb+tj*Vd;Udbt
zH*!YY3dlRIH8HS!)m-}w`_zl<CGcnc{+bg>mE98Gf<+hLYdndTC$@h9x;(#4zclUy
z8ua3+PG>=jN_6h)eA4M6uQRE3xpbH4A4jVitf&ZbeeIaF_@<#KU;>K|T}nRD&0D$P
zV)HBUJlmH!p+*>mL)cd5?jmZ%@U!g@_2HvesbMl}MmuZe{a1P7RFr19B+=Y<UWseo
zh-jh)EW+8i2}3xf#H>AW!u0;{iPGvJ5-CP;7K<JRtTtOlOJNz`9dxn@Rhg$xQHQ0B
z=7TsY1R=Lk6=MAgM1nTTEdc&CsAT3mizAd%LA}IPI;n&%S=NAzoMN_c-Ae|%6(~nz
zI7Fz}YyB8um%A8XKY5X&VaDaE2zJ-|D~?O=p^t1=B1a>7rh)Wc^nt7XjFU)XvLT!|
zk8p75?@_Vno>QPhzmbesR+O!3kxp+Nx4dfNd6PIS96lT8(}S;rBm~Dz{RQ>YwwhdX
zZdX$EQ>c*dqX<!Tb0H|~!=1)tQGe;w6yf`7<tC#WXM<|G1G<>aBC12Of&>WwqSgsY
z@`rj#YW2Uqv^@4ob+I!CNq-MEC}mu5FZi(jqZZ}R`=f<4*(QL;C12WuR2-!!Xg{_H
z*Sq_FTU8oN6coJvNFZZ@D>F{=@grY{LcUaP!T0nOOLMn^xllg481wmC^1JC0Mk3d`
z{J)nf(`!};MY4$r=3F+mZdKGYG&JvHRQF>Vh%}_>%k6Bomfyn`KL_)Przsj1bW7CC
zue$_z0+kn1nL*f6CfI9I60b|>a?MutX0E##X}No3`W!n3<l?(X4tKlRZn2-rpW)H#
zA3VkHw$bn^sD34A*SFQ=%`5+lwpAR*I4|FlGU<7GIWPXn+_2g=nrB#DEu;9<@;xdV
zJ3|1%xOXMCS&(pb70obZy@2Qc2=4N$$CW_C&;WbtjdXJLbS%SOm@la&MC0`c0#$_(
zC5a~M0WOtO^WY9W^N_I)a<f~}aM2&K;Sk19@~FaJ23X_sIUf7wAD=~L_BBo#W+uMb
zkBA7SzgDsj@AIOyTy^HX`Yj|?ZRwKp%m4KA70{Y(KT}c4tFubciEq5m>*<lMZFk-C
zo%mPK((?F2H25_6Zkt-KGW;DwS#5i&^t0p@8Kq;DKPmfS&WGV5po+PZTT>6tY@avS
zre&b?%^BCu!(0C)jOB7>cl~vvu(IUvcJ@@Y{AL2JM<wZt=(@SS>5Kj7`)#eU#uWD=
z<0pf|rwAryO}T8Xexh;NK@12?D)#3oaYnhY4wK1`nL(JLsewfUM)=SYoe3aa1nb7Y
zd*$oZn%!TzZ2>RljH-W!bx4(?G(OCca&kIFlQyYnw1x0EW`ra%ShM(s?!g?<k~1Bz
zR0Rp~Ffvvx5%XeyyY$jvm`2t4LL{qA&YnWWFq=Ru?R3Ihvn=Xluyr)%=dJ$8e2j{z
zc%zHY3Qye>nAHQk9YF*tVZs~!KKx?5K4`NGy-{-e$+(Gjp8%n(v1)5eTT^po3GqEO
z_N?;v6N~=dK(^Q2>sf0~X}2mY=l4L4_;3p#KGjZ39XDTA5!knn1P>-oe|%UZV<C&y
z)YK}{Hn`@TP0po{RkWYjZjSI5>VLQ|Dq6_;%b-vIL9RbnS<gql`#k8Fpo19chdCjX
zm6M8Ge_cJ2mpvJ79<=azzL&Wp?6y=rmjIPkjEj-+B>S9X)zmBpSp+m}j+*&Vb&8#x
z{j3V8Zj>-9z*Af^KN&ERfpq4I;z@wvvLxL}MwJ|V97gT9p_as2F=tMT>1fgDMHKea
zg%Xf%Z)qv`-;hncBkeFMx?x+bjzDO+0PA4=O6_*S2k2CCD1;Yd2?@=rfNopa{U2PJ
zbYv5DQ#xeLIWk50zpd}OKo?f~6jSNyXO%xLZ)A(aim~d9{zjzfe!gD=aNhgB_7Tme
z`Z6!@1|%!0wo32`Mh$y2lnw37nS3?ZqcMbFvc981^b3EpsGJGSjV>CNEoM|0P&YOI
z*tB><RR5T#V0N^Q8qX@zYrtn++j^I}lUqVQR>=V?=Da(S94n@SG|3?->X)!pl_x>W
zFYc*gt2}X-!gG5d1hDC3|8a{jUuQk~XpIrPGf`kKZH_bH+kAtZRVbHHLQF8qK&4zV
zVvSA*Jva*f2ra2mJq-(6>TFy3POUNT^I$z?^9m=0E&@o2=|oDHTpju;<M<ojFyzON
zOBmLG-1_S&(q`U(Bi|CS+oxjFCjZF49u}c8&ySjNNW8zoqReDbeJr}CxakADe)@%h
z(CQE8@_@+^b5+8zq_mQMbxo*}qEPfs<pCBcEG$!2{1RZ>cZYj4Q4Z3ff8DCCKoAcJ
zZ_J5MsG0%T5LqopzA^)r(Qjljec^2v^4{lN_4#s(Y&k7LU~OfpX35*2Se@Fd33={K
z-D0;l*oC%e^YlZ0D$u~(1$AdJW;O-_xc|Jq@@F;<BV?@1Z_vo;Rm*+>-hYi51BDo!
zRk=G1G8&z``t}86k)wh>+kr*Prb9U-x;;L_gb77}yfr!Tc)@-t04uXm`Ku1evzxMA
z{!6#3h7K%8BLb$_EnD@66K9N7JRSGynkJsrkPcb&Q^CB$`jV98IJIkf7p1Hl{-uFn
z0KRF624iFThZJIwXbD`(jI4+-nb3wDtLWw>QkF@D*(pV&a!eOr^Ri;S55b@=Q#FY8
zALwR6N_ha=sQMdMx?8fr(#0{$O0mLO(D`_hnO|A`ZYcX5Q2r=l%x(#WJd?(EGf__W
z5*FhsBknX9!>OA8;HXrl;SxpQu$U{$?290xXMKQt#9f@^!B>}4Z88Xo7#;<?D9K0{
zmy(E9%wFGS-<5LZi#M)c`si~{f1w|*kaas)Ik;rE=wnEET3QZtpdJ3y^@v)LB_eG2
z<ok&U8EKQ_`r}D43z|4H#}X?pwaJ0T=+949A42nU<0R9rD)a-62LdQVf>{_Y^Evte
zOrC7p1=dY4Si@n=HSOQXQl!tLJvFRl^1l)6kCkiR@-OEVKHaF$@w2jO8Ge7>v}&_6
z!`md+leQ}rq=;aJ7q;#Hy-k~|O{m?Zq{j8PrfdCNEB$^nlk!mdsIUu%5sWdnfCD^3
z?O)7dg9g2yTX$bQFsWfSu<+va*V*D63PMVuiwSF)`EO)GPi;6jI#wWo%XU9=y<D!;
zUc~>51t!dU7SsE(qXru4{r2C~Vh3@IwY$gSIAi~4yzI`Q#1T>AZk%pWOQ+i1m^gwg
z#QA^^-8pXdiz~{*@L?$A2ER`S&)Ih5SN~Xpm_GYlH|%9q)S2!I0_01yzK*JyX&_!I
z(EygN*&yt76WC)sXrtcKf~0<QNvii@ALg(jJsg;B%$Cp2-rp)XGI?%$K?~e4N%;TF
zeuRJewuAjECP^X^Yig(Qt8M(xhzXqlq@8SEefTacwQ7+qKEdLIh<TOIfJlEWoGbEr
zYZF2pn)nCWq^SIqxjlDa>(NZDZCxyI7{EnP*O-`iLJq4#5X>jX+E`>0ebe;@KiC*_
z9Zg)|<k?!nZj>&!$$NgY+w8`0rs@5(9l(+sl>Lry=&7V>iPAiRNh^8hP$Z!8ZM{YS
z?r$67)Q;a8kMg8^xQ4*2!VEA^<_E;We=Xwf*rQ8am8%<?;ja-&*%_LTaBcFkyWWX6
z)RZNM8!-1x?#3^}HJzE&;VhF<K`?*_Y5@gU6($z$OmMNZvs??M$_bV~*`JTXVh9)-
zk*pO;*?FP8tcl4b6tS9$(c5p+IvaH$?|od%1R2rar~m#$vzAxz1Ghy*fE!C+wdMT!
zuZDLlLVp;5xaZougd!5O-H1t!`aqvy9x#O#`VPtO(FCFhZ>RPTSCyJ?Kc$}w;L<zE
z!Ts|nsuq0LUN7Dgk`Bhs>E0qNb*AdgKz80t2#nIbVUV#qxj(3~e#_U{U^hyjq?v-P
z3p<_iz`yEoM1m{q3z_iitEF2|Ez-HnrlYJiTfS+ip5+n0YZ<WSs7+ohrCL(IvlR>z
zvKGy<{LQD-01L92tBSBy6%}HQQkTTpAMXYvd`s8fK?m$xYt{RWWqc{jg_Yllj#ZNF
z>19-cM2$V(I8{rpM*xT+ejdot=Yt_*W1EK>fj5~5VUhT2mzD~g9bJv2D|o)x(Z8)z
z9<WZ`k;-CPQ4})``}%*><~^a&*d7344XKW0xaq`!&w9-@$;B*5K%utI?%!cp0TLdq
z%xOk^k;h(xQkFOPmR<OCrCroIf9g-mGl#v;IU|#V2f}SMBNb&HO`Fj28-7x;Uv3(?
z>rXM{Q*Hx8#%&~MSS66h@hHn_dW9I+-lAYA4LfDn*i?NW+1L|<B5cI?sXHIYKV?@M
zb}<EzjMB&v2VEKj%r2HYd~0?)BA&I;RQ(W9vR<92e~@TXVC3GO^1#@}^MFdgL7Jox
z6OOJVc$CO}rMVG|LbNxQBuAeSy?;)DiM`N3vKrPwb!M|%rFHYT@!}VDVNO)0)sP{O
z@+2ri!tl0C>mb0_o$dWudQ;?d(kzTsehu`A$>&Xl((|ZoJGR91BGW2@0po;}1d0F3
zg8@eegWELl^BfHuJ%jJ}WuAT_g`#BxImR@AHFeM(?+P={`JNd&f4cq{8P%=t#(2uZ
z?|GF3G+96wuCcfyFB%ztL{RQMce$?CXl&jyKnPW=asXtNSh~f9`*9BoyVu{3bc#&;
z)8s?B9#`QF>~POnf_3j|!VI)vwAMTvk=i&miL@H;3EmD08O(8oZV-Ba^4n5Aib07v
zFWd>&xv!Z^$$ZTj#$y@|hP^2+w=RzwJ(hB?l;Et@;If{6ph_)%4Hl3ne3@PKJ1w<U
zJ-1NGyg6Q=N2D3pxfISt1cn4s=Hp*(M}ydP!GaI2ht73w{Q?K(wq$fEbV_v0-<XFS
zEoYQtcDa0MkZ%#3Q|{)Z+2z2FWK~zbD;1hG>gq1e9~Wg#iR1TyraORWc#YN*KIO_o
z_jq{juNmKi(ZM)T%~}|)u0h}bH=IUN10i2&@=!gzx=cIE7gu*0cm)*STZ^JIL9@k^
zFCuq^`(MB3+WL{{nU4c<;<o8n3M)(>*^gF3mJh!43_D3O6;aGy0^0;tdS+CqYOk1W
zY$Cycts@&KM?P7pS|(4iT84<K3(?5^N;H1qOBHfTZLpM)FC6=3*W^B|&-2TbE1pp`
zP=#3|G5Z}uMb(q9(#!oggI+u%g&=8lio$Xur%#=TC&rvsn#EAIN08alE!(FoH<~4^
znhfezmhV^_+|zqT*nahQjZK1+&8zf(>w%uI@roiEz^yIE8)2DK(Z)lcczj5hU>W5G
zE>**aw#S5gwD*}~()g+ugr2k<Cws&;SWo*jHhtgj*KfLWSIquc`O2B{Wqkh^rBJ6D
zH9toNwY$2O*!h4YItj8JL^7P3aV;=rl)>~@rOe_;)B|4j6jMe=+Tc2vF+i6L6;tf}
zPO0$r!_{7mL-$gh@_SdK3+XzOJ%JRyP5x&4BkKevKbnYyyqacJj%qc=j;E}Q0$qdT
zmY|l_86j{;Jy(jAcBAr|+jF{VkraoVtE_wOh_#-)O3!>H)8@8v3$@P(UFt&I<6U6D
z{0ypn$_e=JZa@38tH~C&dV_61e)EQtJtoX%Hp2cNuD<D#-q$u1o~`rb9iA=4TG4dE
zySW?yS+Dq`^2Vu%s5B2AW{mZq*hk!9M%-82c>&_%AZy55>zq?+$ach$*D_{lTqQwq
zeFjWVAsWrbVRnrpBkvpE`{3X*A!C4v5S&RQY$fr(w37G?m?M`pnj?NKA+M-N=D%h?
zqc!??(&D6L<NIna6!|@(fak~}2sCq1EV)h7CP~x`zm)A1SQ>Zl)x3EPp`?gHgzk8#
z2a(|Ji3=rPwH_73V($?ZzeJ0Z{!vrZF~cPKO!xI}zam<!Q8beCfnKA@KYMXx!)67T
zu1daVXqo9SFG1si89>l!rFlwypC?zQidkvdVoQHQ?O`k&;)toQ6efVjC=wPwG9xkU
z#Mo0%FYdpO!YqqaFlx~_G-|}S%sH+!HYw-sp9(L)5avw)(#>!VY|3D4GJhffYM-Qd
zq~3@CE2(p_M`A{kZrvjKKOp78c0M3*nU|(9tn%&dzjn239f?RtFxO_fJ9guIiKt#2
z;VV@w=qrCRoAq^MmG}S0PoSOak?1r9m!-H}F(d8&O2@ncE2LYC%&Z~}!RnN2bSqs`
zTS0vCCxkT|+4z?Sq4|_TnqYNaYT!?DO4nqDnmyurbMn259OfM|V1vGJ!MHe}GM{)B
zJ&#N^WA6Z&3lYUzx(+1fVe;I^Q}12`O;L0g6M)S4OpX1B=U2mDMBZEb9?$WFXIS#(
z28~jga#A;Z1t`BOc(haF$**w9JY&1C4-8KpwD+84qTw2kCW=zGe@E#X5XqA%Jsceq
zC%kYx%Y!vE%6?Tdc-BZ+`M(q8(_+p{fyIOAsSl7letx_ahg8w9vid6E?r=Q~upHna
zzpDAa4AE<1U?;+hnL+H2BCg8C*NY+>uvKB>Iw(0!vRezNz%4{TU)gKaW?u^u9L({=
zz!I>_FW_Vhd(rbN@sh(_U@`!H{H*y0%ioljd#ybZAXh8!9(Ba`aez(QS(;`Rex)ob
zxB5=W#m${;?7=H1q1t)T-r)UIp;hxuyMbylfLPkNGV9{>?*KL)(u@wD#Rui<5)oi{
zewLUzlMSW`8=AW5=6d6L{0uPbI&xw4dVpEymDm6>EIQTdvawN(AB53JLe^x`ubO86
zsJs@V=LC@^nKbJ@*PqR)A64iT;!6UBYed@6@yrO$GOWZmw;<6uhM^l{VS?^I`}Vj;
zanh%WpC$8<?-59JVvVNNz(hI03Va#ufHAy|raj{z>0mIV%4im?8OLimMRcPbc!D?P
z4>&-ylFotfH_Kd0;r$8j?$`ay1g{dtYKC94lryxLb`|cVIA?9>9@AjB?L?QRPm5@O
zhLu^bjs-)?!@MSclK`MBpJAE-SxS*{+TX&jFmGI#O)^ZSj-Oh&oN-+wxMG+u)m0Vh
z3Q?mdJZJ=~TokFv1kYq4VniKFPI%lR;nhFi+9Mk)fX{Y3-5bs|0KNwYqcgB}8dF}W
z<9Fp`xzB#HYO$gGf2Kn)N3OuqSb{!PHb4b>)Tp7l+``FN(U+O+EvnM>dUO08Cfx-8
zIes$RhXu8g9W|ej_)q3B#-z>H1he&^N#&a4E`AZ!9)wuzIeEmBeywJyel+nz$Z+C+
zt5PNSw(sj~dg1?D=FUL24&&>QDWqz1LXh-lO}x{UW(yS!DI>rw&9|rL)q|U?f7>KK
zsETPWbWr#M6;JgXWb0T=UEn*EF@c4lOIyKzJ5z)(1BN$Zl(<48VEcSMX~Ay}c;o6!
zT-C|b88M?M6|l2yr76>KU5=|#iE)I77|8g^>)`)MWb!7_<+R@Fn?$>ykGvmg%OD44
z2D3ll#TtP!be-iXUF>hB91e!`T7^U9`E^OudKBw-#a<c?rsNb%V0v@^uA!g8z?SR6
zvcK&WG&~9Ge_`nF&S@G%Y{#vS_nZoeUwxAr@I0GIoBR5*#A^`zsY(^5V^iwPii#^{
zBZMF5T~zW=UoETEGgmPG+x9;4PRe<34}=f+wL<{=T{11&8Yut?Pr+jZC=JAk;T_ml
zyNqDS13OOkjIU5&e};B-Wd;L`{0&5HX$JncLZuYIGY+HdgWh!D6Uaa4W1>J^!W5oZ
z<D)kdD||Vbm}<GDv3Z!};4!Yt6=ga%alS<C3jBZXF9F`Ks{Eyi<t@3vg^dHj#FsW{
zIDB1Tp93<LHZj!`;GZZ`1ur<5cur;Ovj0?ViY6%=yZ+Nsb3j#;QKhVd0Ws1LeXdYV
zb^}n^u+b?)p(eHJb&1k(1Ui@Wi!F_po|EjfEA-V4u@}A{S|5~I7wq>t0vkbk3%s7|
zR=|yz8<57(4{AVNW=6b3Yn?Y0>re-N5|v;rw4w5u>-}WC11B4Y1WY3`Wed30l%mTf
zDd4|vcM}2<LVi9tnu*PofMNEtGsn7HA;|l%z@VsQ%rBTuDXkuDMZk!<N&wIaJx}ti
zv-m^&&7nMgR3==wVSgbG35V%x+Jh=8d$`TE{uPyalOYEYBDcA_QAHQXl+2cU7p7dR
zA*vgVWTJT+Tqt8lbmX#F-tdS<oEyDeICb2ivRocfmgEBZGMr}uLV7#2l|QvaLrS#a
zi(L@HP;!WlUgDXc<_3%l=15&pjzzv^KtF-YLV<h4^sdHwXzMIDk&c{3+PJCyub|-a
zqpLgcriXuvkr$}$7Lzm1ezZlZZjbUI9P&8OCdpS<J}|f5BH~E$F1L0MA?n8h3<E9~
zJJ~aJ?9HVLNV=i4rIS@L4*;PJg~LHNPB^c-sTr*K{B^<aSehSzyH7X`SssV`zui60
zmFXo<-A!5RD(Qx52kz#dtya*d*bHZ_>l&{`3Uy5KKgnM9%i;2a#&cBxB(v9*veL$d
z8oN=nqnLTW2K8sQ)xdSx+wd3)(6wM<XkdWRA03$@3mztEFI}$!U|S*RfVcL;9DoDq
zb`x?PXqm6)7fwgq^y2B1ef-Z!U)Bub*ViVKiQ&0BU><Zu$1sr|6DZ(OsKXY%)DO!2
z<~0Hgfe%!y=F5lwQonN)=uPs^6;2+^L#gu*p_+?EY>L6B2iqoQg$q%;pSG5Z&|H)p
z3jx@XeGi<Zi)k4R#fIUVajZcY`txS&r!kUv9L=?b?R2gzj0sf((rS`@_R&5p>sTM!
zQRXbuR`F9&Qck@PkrnV`SM>g;>>q&tbG+lL7y^Ec`3YHCT4n+lk4*<#>sqk@1TNX#
z0Y$IDYcc!e7rqodx#EX9luchgc&2<0$gJipTOoNn-*$iv+PxrwLeB(rcM20jW#u0e
z6H|u<A-X|LGMOuMur<(uu5%is4kB$P-Fw#Fm#tEwXvKevCO*T=i2$1ZKRy_VPHRjM
z8d_h2!+Hn~JyT;rOo1V=0M_c}C4Vqh0bUBst>=mtH~yc8Iu<Y85M~KdXTU^oISYCb
z^8?QU!TRGkhux*nKiS2W!v%9Mj=TXtPRNu4#JRW^&COF$<Q&gR0KdTupdQx9t~gw>
zG@}8DkKqAd=XCiAj0qTvDmWMl$q0r|?ms1wR0BTD9lqX#^8)R6;AtCY1YaPjN-m3;
zr9|XPs&55gh{n&QcH_kQtT#FvAYOYD2T#zQm<JFC<1aAxURAY2{PcXgZcyffkZ_bZ
zt|40vlk&bSbvIbV0WjtmRuw-%Im=W<xmUDktAxg56HQG2t%~cLcfccOdRw8fFeuJ6
zMCl4;V6tIx1f!s=q9c;7%6H%C1dntYgBNBppj3R~T-)JF=u5AdmJ-}MYybajh!(K3
zvjIptfY<cV^yyrvHh6~ZD%UJl$RAB0u5#jJtA9M2$PK4#M)uM%zfElcT%pukKNm3M
zR1oW`|1?q;5!hRG80G(*)<-y>pWQK!rTJWGMv0>`R#8F@rSm0HK#Wu6mUE<?k5zxP
zCMBV{0R+acSP*_-HKAZ+?Czc4KB5WB=dG!L1M3`IT6)AM{Vq!JO#tiVavTl}d6bVn
z``t#lC8{{;wWhhEJx|JyYCqlH+_u`=`*C%w)894<8yeHe!rxYTfzxWK&LTl+G77Nb
zx9b3%31xGr31g$1v-i!aI<jx7lv~5GDj~deB=9dgZa@h8dvx!hanO!FY#%cHsZGMi
z4wl7{nYnjj^Ob7+W|Q?M9jcaTK=Q$x&g<yojrTv*G$#X`$bZxifD^%o)pq~ayDPHT
zN^ZHpBu#521<3CRJ$bTF%gC78GE~9X42O&pg^BmF3W?le(wM2L;OzOY{;|LU2yk+@
zS!MqAK_6hnwu0uxhP}WAj^y@0?WH`{nV;F5nKRF4Upq2fwZ&z2iC%dwy!jeE3+Svb
zJr{7#!jGc9MJ?ihQdr3UYWn%6r@%$yetyHC^kyn*>ahlYYjfbJXjG~@S{P$!Vc5Fi
zP+gye+*@#8SGEaI&V^#yk^&3KGx4X!M0s2bA_EK+OS*}BJL3Y;A|^gr4qaKw+kAKZ
zwGvxu@=~+UJ~<~N`G*CzN%6yV1)~+y;UOY((T$lCt*~}6`oyvV3Np!b*EGH_N2LFk
zFRA(-gnJn9ts=mC)N1q;XbVV0WJ=Y(s;4gpW?1wcF~ny4(%lBg!i<ZaHyEM;1tP+%
zC<V-U%)a*T2<5jAKbmT!SZ#L+!btY*{R|sHjFjt4{XN?VSruQ!N16qOn-qiC)my;b
zfUjKL@*gOa+f?s`k-BNuV1N+>OYr|^nPEY(<Qpil83VR+%6u6uUQhq}g*ixK^J|lf
zi|J!Nd+)zlk#zo_-$x6SE$D65T4&Dgg^=Dek1urg=aBd;IkuT??I2=L<i@^qYx-hN
z_z5v)y0Uvtxy@u-WM#zMc<@$wl=d%iBf)rHTVM(>jD>o$9=ObxpAGoA%YUjo3}mwa
zEcbf{jMaALtAy0n2`0@C$FfLxj0~co<tZHzfng_d3rT%3{4fIT^@t?65<;l3pT~Lc
zG@12)W8YWlvka&$G)#oR!S5=)_yA^WH$AG4`McW+HOZ*N(~%gLv+{W{l5hl{S-(nX
zd5_(Eb;zbrc&$*cipU@o48|}I`(F?timB@iCmOwF_ZC1+fP$;^40@n&Ls3l7X9<EC
z0oBXrUGj0n7IS`F@;cb}d|Y7<ndt!dVx&vi%68wXWt|}W5WqYAe~+jF$)xk(l&UZz
z<&-W3vS}EzDP9T#^+hBPu_#S9s%C&=5?jR%&v!QM|GYoO;*)}A1aJyJ;`%HsEC_Tf
zZ-xxm;uiOj9+LsV{`QY0s7WH~73c_}h&t&UWG)QLgwWs31I1qPL4E6h1X*~1ZFVKw
zdW%Rm_CX-u#~l1Wk>+84GTy3|#QCU}SH9wSV$MKrIb_swKb9y+a_v$<Lv~%s-+y0E
z*!iDLC#7zBqIiqB3ng@)?*1wFLJ`tXQYI1gED{46aet1T<90d&x74H;nIc;{h08F;
z5ARd}IN51Gw~{86(j4FlE6^8Km~Vq(w<Lv=v%JL)o32RdEh!>EadPOA7k&CuukiCG
zce5|WE1jA%a5yJIh1AuThPz|vvC>98N*Zm^;EuUd)zHidJ&?tm@1*_A?prxx_MP@8
z`}R2*)SwX=2^!90G4U|eQE;{2>hYgHLPyO>b1CdEt+E3p>i7)G7-7nWA1{1LB}vH2
zpOS-y;vKT>s4f7Ob;&rr7>m0zUuhs}_x2-;N{fq=j9Q5XO1<yxm7M)8jd#Q3T9|?j
zfHE1?<Z6vMg5&9GS4DR&hxLhE<CYDkgJ_BV+bai%p}}UAtFrl_z6*V8Qu&f{vY`qW
zlq|9{l7t5yf8V3jNU5oosE@_3NH5cD8aW3Xd*QhUz3uErpUc7s99&Tl4&j)-3{eP$
z5gVIhSo0yR7#<S=(D)*ELtdXPc6Mv_+7lpJaRetmxQAm7T!Sc)><FF6k`s<Sq%{7#
zKd`h68H82EOClY&wODXsh)y__%`=!Bj3W)M;zx|tbaN#rdo#;LA)Wt3-9nI61J-4&
zW-x1$lX+CMuCa4M-EJg+=5*tDRR;OB4G>_>UZ3BoSBOKs)RHzz!!`AC4OHaAsjN}#
z#hO)<wq)<cluZ?q8R!Ei<i{54n|j%T0EXRgV*b`Wbzp|*x`6JiXmLb_sItP9wAsh6
z<y#DL&T^ZV_v&3&w-*FxnAPdOs1vX^GsiQjO3h;&pQ_)gv>k{j)h)Uz-<D7>Z#1Cz
zVg0gNY3!@&Mg!9^#_t<ivWklpf&?{|!Aina6#5FE=$PW4EOzpjPAfRYW4=|$ZtjhJ
z0P;8mP#T}BxB0xyh3(To7xf}boTYIvNojh;?YvFFH1oO)o0MbYl(oa&98vCJkvy1@
zoboA!w#u25XpKaQUbQSQFOQOAh7A&))+b+UEZdCh9IjHXE-sguQs42TP+-5s<cG7%
zS^OaLpv@CJX+fG_T*hxfrlZu$NarTk8lFZY{17DVz<8%J1=4z6@|gYR^R{{fAAv_X
zD*3SBnTdk$0(1%!oV4SPHz`hnGn|!BH|K_61c^==*b8i%vdh<ameD{h*UndjaA=uQ
zO9g%M(bsz~_VZq@%DVEi1K^h0ennyYsi;xQ*BgvvQs_Am4p=YUSyGC=!<;8Q)m)oQ
zB-A3K;e?qcH}91sc>2nEvGp%ER=i(TnffNP<UMLz<C02qm&U)B-(2bGk3awl>fL9-
zfj1-u*Q&cU3Pjy#He((0#^Qezg!O@mf7v@e7dBju$`KpQ+(iAaGS$DA?G&j7-;ejd
zM4A)Egd#G^O_`Zar6z7tMbl*1xPbbmxVX5^`AnPcO#8cp_OAy@;nCeu^HQUJW2G!1
z&!u*!M70xebB#W9gM;9T=cy3sg+8kll60MhZ)hP@<_16`=x}HkGE0IKt63^KM)Y<>
z^D)2R;ED*mjE0?3fB6Er2D)KJU^+@--N|q${XO4(rj&bhGS4c0VD1#-`>~x2gE%oa
zt+Ld2&Iclz<?PSgqqqd^dhcL2yjm2&Ht6D1p#wCWKRO4PvbTNMzG2&`w7td#76FMf
zMjQkMI4)mMfcvgA^LqSgcGPkk<av;NLy<z8Z%)KcmFYexb8+FqPnIh;QZpvEbz28B
zt<d-e-sY^P-cB<`SD%cN;6<f~sfO>B-JO?A;x<ORVS;{3<(O`s9h%y0Dby(cW{TC6
zd<>{+5A#OF`1Q#*XuUO4N>z;1wny9_ec=S500F0Q2jhDnQC2gm2L5cXKjM43U?jp`
z6Dc4NmzW(L4bXsNFNBwMKMC1yZf;hX+!EiPsb-6dNfX(VOD5@s8c!NDvwq26o?qC^
zv7LgKel^dA$_bq?p2LW!&_#`FLYL?i#^{Whs&^}9Egg;5FuS4@c1Hn;`=!?gAY&*q
zozMvyi8o7XeC?j5!S<kzQFlOMhqwvbh9X(4tMp8#16O&satWq?nco=Fj3Q{biv7fp
z)b9V**M+!{il%XJ<<mxPsytb#1Essac<oJ*B>ZIatrgg~Si)=9t#Ra>uf~lCse&*;
zDo~S18#x>MFfm8}Bkx<pFp#lhypsNB#^?P#Grao8(IS^di;A*~V)z?ndZ9?75xOZ8
z#7`D_z`*m~xH%sfE)B!U`|VCi&b10jd<l&BxEJ61UHx{7iTortboL@O6Ae9^PTR3a
zcat0*Rqt@7*kyZbOSfy48g!}(ue>FqhVdi#w5F-_4Ifr#v8_#T{a{N6{?U35mdBvM
z6LU)s-aFblBxh{ms$3;Ev|^j@C)kNAZ&GnjT2aP@#UMUje@`tlp{8CFiR>m8%mUE;
zDdklC4I_X1j51+X@lTw{^%)+u<nHmU#S-gH#pTDCj%&Q6jbP4hgno0RvJUy5AW((~
zY3p@q(@@q%g}!|h)!)1Y+c_57U|}uqjlRA<Cai$Urj`~t&LS+lAzWyTRWZG;)mDAp
z{JgvgfKPHqqRNsUybsUW6YjDNmyBq2yXz6hu(ni%S#H&+f_pd#F&2&s53jb=MAuyU
zfGHFc;6)J&S|20-T%>!cnMr*wz92XpHUv<7OQ>1L1FmUA66$>@hoXS*d0?7w$e<Vx
zqg|nfIgPEQ3^tjb@coc73d}dKZ3iY35)Nz^5gn*<h<%+DC#?jSb3#W%WCx}NUO}1R
zd^D@LmK<LJ7?jz{a&B>foFfmW=h88%sbz2dZ0wGwkI+spcJ~^{<o)r<m!FGv)6B7&
zCXc=6zwmuNVn9frni5Xrs?-JSWtHPlz6_&N4}@E!+|GQ7Kv|V@B&=sID*n0M_ni3&
zKZ?s{?Nr7V)mO$QTc0Xq>KxIpIKf8t;C*OPvxdsagvNLb91<>_P+OowY_Zqbx{K~$
z1&Vd5haGBQ2(o}El)Z`ng5_=*7H+yF3(Pp>yqXsWACrX8J7};RdI)~NO%%@F%QpLm
zRz49h^-K&+Bryg<)7o|FzLkPM5+PYx_%>YE2_KMH1-a7OLi=Ib(rYO+fAT(_`&TK8
zH?G6E>}zhj$Bpfe>_;Hb#O=`4hz}h~hT+PTEVD?j#}c4q4OY`qYEoPL7^^9rHDtvK
z<Kz#;8BmTYTUQ#zM!K3!KDOO(E~fnCBPZ#v5YFSNFX{F2F&YoHvGCJ*@tp%6?5%o4
zz<t3W$otsoY&|&;9-Y*!IxB?_Kc2p3*>A|3$HkuOH2=)T_Cjx}$$}R4UGP#jrK*)!
z6Td`f1V*RzigFN^QJ4m=0=;OwG<nV+y3Kam@vuQgS%y$a;<ZvPPNtgV-x04Enb~Y6
zV|?k^uq0p<=qN3F#v)wBv4OzkGy+6?JXJWK;3P`rrRrM=-McSl(CeB&h5ltnO;Dm!
z3^uA&FSII-8XseCu7^pfIB*o3&_921MiwWLDJ`44SNcaX(&^%1t|8ldIBjfLXPXAN
zdw2?6J=g`B`EDp)`Acx0Wb4HW`CesV2{FI#H*4$r>+>)6I56OwMS_f-&z_nK;OhG#
zvkGLI$Y4|xZo@CrBgVk#e$c!Y-RLR_go;(1&Dse>5rg_SoK9<))UNl3)c#-K=Yx_@
zGXHOX{TLb=qp!T%H`GQF3&J2|V?ce4aC<sYzYwaVc(;8Rlh*@_N0pjP3b}NMf5V({
zU{=L1iyr^|((E-}2YL_i(D*+9Hco|zuC6p;C9LC{v-P@sDR*k)d$&iyZs%<UN+QjB
zIIC3t=m@LyZ*s(I4Naa2KQ{EAybM%wJ*b<^vj(g_!#Mdufwm%oDQn#|t+=u8FwQv1
zbH@?&ddpX{iodR>Fo!Ok&uZ1`x^n;Yt4%f%4r0a+^{yYJ9Z<hqvO2hL&ka}R$aR;p
zL-M#ih}_XB{{S?f<5Pas=gGh8Q!u>y9mdK{lorCrI81;2FZ=;g5y+5M`}NSsTBD`m
zw7|&|dS<B6ucr*?YB}i21viZ@)d}oggaX3?(8XGO_3wyQuL!lmR)kyd+T^^vCIU};
zC6|>Xr5%Xfq}z&kCnCR9P-S2>4uTdyKJOD15yd8>snmNe#QAhQVgS0T!bl=&l8yqk
z5Z?`JHI_^y9NH8G;1-rK;{-Uec3pZ{6i2sTx3lOY4N{&jU8?MtdyCi;Z4{&07rY+4
zC(_NfOzK#BK=a?u9}0JF4@MFs;ug5}di84hGQ@pBxCgPWfOEQRF27pDzGO4=8~Kj^
zk50J=oMFuHBpZRb<GEtbtT=OE&mA$DVbcdNsGS;DZJcUdU@8|)CiOiD_L)pULGcHC
z6BAal74d9_l`HJO-J&(T=1LB<rN6=M?e|xdQV{98&hwXac-=LUl9f$ZheR643b#Ru
zFr<FDGjG=@$eBrBYMLR>>&mw~39#pm@~Y;7NUXZPoKIqhL6g*FuZZ5r9@EH6e<`n2
z|7yn@<{ggd?bcmwy0`F~kRV=0d8qlvcQfzY@r<mgzjO+NQ=jLI6P}mnu=MwCTweaB
zqL$*0^W~Z_4Vi>_3xmAV)Q3}$k@rrC+=^|ZyWtNCN+;?eM|Gxa3WpQ<L>_D>p<(32
zvwWtuw|aGkYl2rdGhA#H79=BBJNT!Fq>b&Kj(<-I%2R%<!aGt9roa4=c>vfemS(Rq
z{#<VsGJEDPPVXKvT731X6nIDg*a0UqhmDOkW&xfYUyj>;X3y;P-DktEY)+~YWUhOW
zW$~~zYMG72LYr6$a%a2m2FgWb=}t~5#)b8{z7sMcGBIH@$w$0_-{V_TCV@$lsot)Z
zULhX)4HC8XvO;E5cdyv5F3)iZUYK72zB|0c)|uTVxnj?LmJintZEYbdK-1SjyDW@m
z)U74E)$`gicn007S@|lbmW<v=1GS}1rW{^2zQ<)0U6c~D4|V_J)f{W_cEN|Pm>pkD
zzvGCzhfg@n+rj|dmwfV8&4=1oAo1h$Ujb@LB>&N*w){*N#xNrzrw({3H4zl=QqI=i
zo>L1Xbke2K`Yj;+EYU`Nk=?RTZeZtx+lJ>Gr1%f3cL%az0I?e+XvT2#p;Iam(sG51
zinKYAF%ZDAi)*l1p+!U-_zDbjeDFGE!>OD2&DgIu+%6mNUhMu2n4|G5qQvg;I2d^~
z7x%4B_hx-DVcPkfq1|259_8RHgGQw=j~;(%|B#C~*sfIRa)cBjMHrsKkp}~Z6eS>C
z^@fBu$hG!gzwX?8&W}9bt=Fgv#dk+5+u&hwADNKpxJQZ~8vbd6W#w0tdRe1e#Y%_g
z9v3Cx$lscNTC1rLr7laWdfnjx;@$loBUZoH(@|8UA-;6w_PPAg3_twgG|3o;K#)K1
zh9O0S#)&!Ws5|FzW6ns8`TV+LP!ZzlU-2v)HT9T$xZ@D>Y2aX=L~f!BN9V^*MUztB
zpG04P9gMyMEoYl|Pb$XkD>!vK_Yb1shn>63@#e{lUd5s#NOFxC*h2&D&Ra_LCK?tG
z?~Gr;X35w&AT5J*-2nC@EHo^NptJHTR)-&oWC>zSsusAqrE%+EZ1h*j?n$cIR}*ry
z>E9C`Z;VfEzsXpY>$<*u7ehGoZtW(3^F>A`*n!*3ya$NGF7RI&G$6jz0H9?>szu&P
z*psV#RqD9~g$(1P#oL~YwwLSm_Od<_Zm%UUvr_a7M{vXVJ^*+l#Zb;sR*qVWWowtU
zqGYdvqa_+IhK%nTA~o<7J^}b0m0P{jIz4qq2fY2#OY0VU-{@_l$pfx%#{az>yf@kL
z=#W6Ii4zs<RAhYJWFP|Hbx)g6svb~_gS8wHHQ|{upAt(r_C;o&Tiy>;>LmW0@^!s8
ze5jRZTk4KUtrWs6`Bc^tRTpPS_kXCLHn*1(h!HK!9bc_4+R0v(ZqI}RLK3hlu)__1
zf<yJJ;1>h5G4`V>bumMLai#BP8VqMT+--7ee*`Cr-)P><pT|DTpO$>JtL2I{bm&I_
z5_jD*&`0jf(*g{7YIGUu5Z%R+I5(>%%Y03?Jq@!C;k}Kpd^Y=~;bJCEZU4};-Exl_
zYr0Tuv4#jQHn2Hc3gp)*Hqlc3Z<wI)_wA5^XXr805h;X4Oq!_Rw;?ZJWFot6gzgab
z=KtyLtGeQ9nzqqo2sQ+lz~B(v-QC?C0u1i%PH+kC?t?=J4#6FQTX1)W0N>otdVj??
zC;Md9?%my0byZbYUv(iP^5x^_RI@87-}9KK+pV+<wU<dd^TeS3XO&uaKO{n@U-!=!
z@!_hQs+%oD5ezS!h*PK4Q{a;!rhr78S+{nrsEHqtQuf+LwBe1+y^}+2FhY%Xhi(^p
zhHPPdBI>LIIAOBu)&RjvQx2>9?=^PgFZvYdO=@>ckEgF3qvJR8$A-=Bs!7QMH<R*1
z#DYFbVaPDM$8BP>Utor`H;k^kC)I2FrLXOT;L?T%?z*2H`KECeJD!@EF!WJA`29?o
zB-W?;el26N6($$bf{2Gbr+P=IqOafZX3_i_V8d~iB@+*&bT#K4`kt8>^!9?BSVGwK
zU+z9G5N7s%Pym?&J{U@*dlEA>*}d^uP^<Is@)wCZe$;Fii=6JhYm5STg>;~9i~d*~
zZ+<2+*5}Td{Y<|-6z&;q@u<iq4y~oU{eM(b|5&!X^J(WhD^)InI*So;7wLR{$;}qh
ze+j|+FI!THprs~sW~LV>cGl%WR<^Rkx%Y3ntAADrJF8<&;A_ULc#GTR%i|_r-rTYI
z35|~SrSTrez)HMEC`Vx|B*>s#trczg-m-=eN$IXu+qmPN-os(DFz#(0e11zp%d#b8
zbCztGs-LGRRxGmB6E$@dLs8UjHCBzp+7N8h1vmTo(}|tyBTkza%GkT(mq3a6^>*_(
zF)5$b1NmZtSnrvH3ceAo1Pg6rmTtP@r~N%lXY$pyX%3#2lywD<3(X(5w-_3jnihCc
z5Cgyj1Uz{6o8oQ_yJ7Y<#{RJfALFO{Ybudu*!7V^n{~{UtZj#FX@k!~tCh=2x7P{d
z9(F*JoZyWTds9t284iV@hQR!*m<jlOff4tHemusaI|^Qis^kAid&L#J6_VPu>rcaw
zfDfaQTwWb_J9aOnOHY|nL_24OYIW`RgC#rag}!bRW6s<C*Ek!$>vuz)GY@EDkZu-I
z)Ea~a$O_nl-<(JYF_?A4>c(*XW9rp?juPo!(t8?dv?VSfkdO#hL45eC-sCCcg$hAl
z)nv>^3ks<ylxmJ~j&lsGQ_S36{0ZwPv}5x>E-B=0Q3KUZ3yzhxJ0mY-!T@OH<#$&b
z8a)U~v+Zj3hj=`3|JdV`O6z)?|I-j<?cC;ddSLf`j#ioSYFv#~VBKI91E|?#hSd4V
zAIgDoyS4j1!fXn?fX%sNih|5Y#?eFlF!t_nA^X?1a`oNC5^oh~dOC&`ej3BzLm?Wf
zDN8dTlu~>+zv`*BLaB<S@-^zxVnp>`bOC5++Mq~dI?BoR@a`M1qi;F6=5w&-Gi7k#
zC79<lb*<NZ@VtKE<v(?OE~kmkVthvVd;Y$pc2s}a@`Gl#>3vQqQZ4n&uE*fDoUH@F
zeIH%*#5fU^`_3=^rwnIeJx=okvr`VeJ14LcA3y8&E)zn18g`KOCPol!-g&WTtmePf
zOUQGqu#n@MXuIRvXul0Bgj65@t&+msZSlvLkHRn1GEAa2Skh?_o|<V|nL%pnbNC_!
z*VCm_=GC(3kf*5`@>kWFrxu(aLILj0Ka29S;1e=GXP*$UzP&PGwH^5c?2EN~vfO+x
zIzx=soV(9^m`M(~xJid;I}$8(@VnppOmMZ0FblA{vreJ4+7<p)X8EypKWA{&h$+S^
zPcV}u?{W>~=|07O1tY-4?(*`xD!f)xH0w@pO$l9SDaUZvDKoM2U9QwWL~<@X&%JUR
zZ>rq#<8*S!Ih&e*?HGm2S8**k)9;9K$WsluVl$x0LP_U#AvP_H@siW>1n>Qy)*sw>
zg#^XyVoA|<7Hi$Kpv1*;njQD>L3J!fjUgg0u$X|A4?9n_dN{KlBtj)E3)9T}uI;Ef
z3N43V-N{VA1Tt5Pw$3sCx9|qaxxunT{TQY;eLbqSMa~@e`hJ}q&I|%Yab@ktCoDI^
z3fa=KdOCX1tLLhVxDnRsJ}sleze_u^f2H9Wn2HfF^*rNuRu&Xi!Fvt!x5f`|>`ylj
z4a=s#tf!(*db6eSpWV%IOc4E<VtvC@75^zt#rlWm&&==6mW;XPA~8d@bi|S`V!@^8
zj|ks4y7nxet@riKA8M9;+O2UnYgCk|X9~UKetdX{m8qo}V$<4#_xh49Jj?SfU&Xi(
zme$!olk3%wnk|4NW^W>AH3@Uu+Uo)6`o)%Sd<z~jYWA=xLbzB{9CPccdAwV|B=nS^
zNuR6KY4k2ZS2zx3puG-J6&I%SHmLA)XKpY3-J4m{O@eErL`yskGfdl55td#@FjZf=
z`94d4odSC{h{GheIAE@bj_}Z{#YJ1jqAwLNaY0<O(o29QA=?$4u|)6!_MZY<kI#TW
z;De2s%_{eNz)XA%IbR<B1im^Y^7~cp?wfmX(xBo9=MS8T<C#&3<dhGLs|sAWhJ;)p
zr2+S-XUqIjDc%R2Oy$lctO!ZSv)4W3@1=Y1eo7cePsmyo4Sqfz7)d7|)Xo;x5I5EG
zB8>8XKXcVjEV3O_?oN0JmH4~})JA!Vxsu?sN(zMd__*Z@KCCH%Ix)+}HN|;Ta0k*P
z4SaU~44T|yB0NmK<T(MI+3xZS>{p*uG+g(km?|y?_>u+7%5r_8O*4Qfo^@Fn7JU#D
zdF<?g0fYCwXzN5d0T)q>HQ8ndr*Tw=le%c@g&nVEPsIaeW>1^$6C*k8#^?A?5+P@G
zK1-T5>uEB_XT;wyqc=MiC$<2n3;@Ps;W5W=Qv`cK(NivE=ybuWkWD^oWE<k-!<0M}
ztyhzTmB&oifYh&Re`FEGm8OP^`o8DzI)yU6CCg$vR4ZyR8ts2WCsCgGz5hd*dX8m`
zWrUf>QG)rHvs&RIKC8OrM{FoA0-~arG|uH{`J#p=jc0r-cvb(kfFJOEW@|Vv0l|g1
z_jHv|J|{x}+D@hT^!wW~hkm4ZU?@=F{7ljel}Y0KIl}!B+Rt~Cy%)~_W|Jgg%}k2T
z$kw14KKSLbGXx8tnE8>Efb*DkF4IWMPS@*;3KppQ3xD?63nb(62ZPbzC?kU%a_Ofm
zuIdspiD;FYU#JyUV=X_F8`0*=4f0a|*^X^{Mt|Y*6Gz+;)?XiDBz$9M$=&%?<*tG=
zilmUwJR=@pYl}Gvew;%vgVOyJ1;X$p_aWL&GWxhp*d$fmeXs+u+krSe1ek-n46fk)
zTD>vX7a$p58&1-U<`<l5vg}=fstI^ti)fr!O7Q0B4K6kU5*#W^OGfwOqxtAmG{@*f
zZhJNw5r3hyl9j(pNGWg^cPP`?hrmyxtpM(*b-SP~^d48-@@3NRxT}<5;dmSKwfNFB
zm~#ea5bC7`FX~$Pt#T8e?S{cD-mQRv&G>mt<G({&-+VS^;*RGl!cEb19(LSIMm}tS
zY94MoG$tlgdPaK7)2L|WH5uP}R9f7kKVTW1pn1EkxyCoQdE$7I9?f@t@nbo|mWrR=
zZtXZT@xD7dVz&jXN_vAW0>iuXn!gkK`=Wd?M>kYLV3O&%D4Wlt(O1UZ?6o+1rrlnD
zIaf>-kW(7myJ(0o*nLubazkEKm!P|W2kT%IN-$ko48Xyk(MeOD+e8C5Mlno3WUkNI
zmIiV;Bq&JF4+?<CwzGE>MO{!1BCQq9)d@_Xgy}vQ4PevKQj!8EcYiJoD|lce(QobK
z=Pw`m*s%f8;XQ}gvP<BH1rqEEiH3-QMxny%tNC*O#unY?$fw__%!QO4?9TMD{;!B1
z>w7fW8Z=DWe@Z8k-S%gqvfIs~91C${ssSlO!#_2(1=IZv;X$u@`M`6R#`$nY>o9Fj
zeP1HRGYdDa45Sp9)BV-A;^`+Cpp{??iv;?0f1JH{|0eM956_A_B|-0cmwALNT+p<8
z0Tn92qSiatWs;-AW_tsQ!`PtRSAPi0u3@UEh~W2;v^-6FrP4l$hAeEXVUO=J_6Z;H
ziqCI`LV+Xm%rM_=SISe#+AODXwe*&ztAjj6jrE>-`KIV%S1wg{G<Xs~63!XZR;z~~
z!+V#kqb~n<)yy?qDHICuE(oMN{W$~>{O!xiosmRjy&ak<q8-_8!G^(6KT?r;UOBv;
zN`mHXkkk0nt=sWJTl3edS9m*?LF9?u?5E9&7UhdFjXVp|30*HGp@O?X<RCg&uQTrl
z<>QSK)Ml$?<wWlo>Z7NixU>6;Jo<c=TBVYq54Y;yzU+iby08A4GWg7G9LzbE7V84k
zkAa;+W+FaZR9mih>Fz@y(C>s<jXTSto#4ymYkJNd-nBBCGgIm;%sm=MH*yANPBgLD
zoU3w>00R`5b0c2dwd^2<-2+$KCF}8_Q!cnf?S?v~b1u80Qsce!T##I_!!*8@fCUFN
z{93pHgsZT*q&BO-<ypALDZ${C<4297n$2BVH)JzPEJHo_pd=di@NaJ3QJ2URw9xSh
z8LKdB)yI-LoKs9B=DrvY6M!y#%-u5arB|ocUZx}VYlFgGhR|f4*{FH2$1h3GjhyGy
zs6OWCkhlKu2WCDzjT*yw5pQ^oA>&E%bU&YvTDB~GEhfsR>j9v=w!T%>`rT$wkqv0$
zoMHn5JTb%7v@`~&2&2HLo3mSJs2HOXJ{BQ<5qySe-ifi&rGT%28!YuhZ?*|~`<=bw
zVJ#KC&!j?u%ft!%rq;hy;D=^Tt9_UmKF>WV`c!y`W$sL{(<v>)^g}3A`uv|nRl0Dl
za<SuNFm@Dwv;B;6q8=D5<o_-qp=)f!<8p60)WXta|8U<CJJ`XTZoTlcw|>5Q5oKFM
zG#P33lPcfIj@F;282QPn+?&c~J*=w`O()ygohX<PK;OQ4BkFp`8Ln>oEqMw4Ah%#T
zFBjp+d>8vPYu-}#58<Eruwj}6y25n7Q|xG)hCKuxiMkPD#tSthy73%N_EG0MMDD4(
z?&w9#KK0`Go|QgK7nYoqTejM)&OFx#AHw1m1ztLA{2DSNi*lb%t9rXXSJE`_Itjg>
zwCO*b@fS$ZG~oEQP2!t&x!9#}&&`5*V)<cmNWXa<5mO;t0VAF6k*s8i4)?wdux5~(
z1I$-=P3Z)0`F<$W3AS@gUbew4=r{h@5-_(Icg}8EpFF}xQ=r4t#Wm%GepeqBW)ry{
zdCA3W6gY^d<C)|^uh!eW!$U=W>SFduzie;|+~aupP=eyly(){nByV!^d{Jw`Mwm?*
znqD9~oaf(dUNOgFT@Wt}5(bMf#(fbi9gb6>3nRJXSk)@m&Y$imh--DvQv7JPZOVLr
zG2Hc9GQa^_uO(5J@_wO}r=@5S*@{y3`{rP>O5S=H_(9KpLUVBmwo8t!%)fwf>Myg~
zBipA)u7(0cz5Klo1!s#kpysVp`!t?^#0>*m?;<}1*aT7$4YKx5q9t~ndht$5_i{!h
z7Xw&V_n{Pgl@NA2bv=x)>BxW4E#Ty|Y?W6;6Qt5VA0kqyvoQo(!6E3%Lod7OwV9SG
zC}1vN5w`XmFg_|`N1)QI3?2L)G5V@dFX>#cKF970+Q5Ah3ZgsOXp42ux87)id@Zz!
zFi2_LexAMXS7-VtxvC9x1Iv~9v*uBCJXL$^z$`13>&i@EBWO07aPpx6w<q)h3eZa@
zH1^3Z;t8!<tmF1BSz*sua|^f<QHjqs$%h29v8Uw;E7z0-SM4U)5vg#t=qI;nOa(y!
zU4Y219gbVgRbJhF3$kO9bKB<>5n}kKmBx8tSV=(c0ui>C8xN9-ZbAb6xzmup8_$>G
zy1aWVnNPJ@8eR#jyl1QmIDh}OJa+mY^A5ti=zCnFzo)&Xe3=|o?-0qG7&l1C-)MA+
zG17ykSVEVXRcH<%^pR1i?i_f@?p&O&p8r_ljqhvnvZ#k9ZJiC~10tKtGuaBOSkEE-
zBPdp$iG2@?@DV@ym;M&St6so9RMz-vxm~q$U49>buY!#JsM21TXP8@D-Q0-lUfq*V
z^Uwr-^WVy>O<6`eoxiEeSV-6kL2?R=Z5f_CjyP@(e%i9NQB01mHl0NHgc5poEqU*_
z9X>`RJHGn7aa%D@izl18R>s{I#D+>K|B2X}UM7*<7QcUEo`#^8!|-9BC{zjal?8%7
z)$BYeh(XzB1gD15$!@1Fx7=iEvzfAwDeke=YNNrCZj3^}n`%vQugdgsT0a<Xd#|sW
zU+~n;Tqrgz?%s}&PVZZz+j>gKGSWE1WdrM0tbDDk$6ifGTBdjJ=)Nd~ft%XZ#vuL~
zm00rOAt1(hsAL4O|Ahdj_}DgtP_H9KlVYrF<3yu9NS+OT@J!Q+7-O7}^CzeY$Q@@z
z+%mmuI@96!rZg`&x;9jgok;xIum#jn5s<4)I-rbOkNeVj$M!lx`a9uD@?tFzE52@3
zLU(vOds4+k)sa)M>2ByMWK@6aGDzT0<9jk`&crF1(2w|`sbZ=c0@W{XLTK?;iI-A6
zd7N#K>H0o2oR#-tE<?xpu(=avct9u)R=s;oWGnInd-gV&jI6=I8>Nnr)^AA<DrId?
zIaLwosmA2G*1Me8q0?s7MT6*4vzE9jXMa6E6G@ni%Y8?Q;{2-`w`3bQq*y|BXOJAo
zee&Dp+a?j|VAIRlDyK<VGZRvr)H}gO9#BRYgPU4!Q8&lfug_JCDN}RBV7U3up8%qk
zCQsV*ho`KlV#sD)2o9r8)i^(n{}WoX&7AgT)7Z}I>H+O6x}JWzbdQDC!Jv+zm0`1D
z%lki}<3BEdAX~<#X9IJ>s&(WDrJb>lxe<-H%$6%wKgOn7tUh=pO-&I%dc%4$xC|N!
zh!oXs7d095e)DWhCE-ZbzpSHAPpg<=EpwE1A|egs(Xn`UTJPm6gq6-`;0f3|LB7}L
zZtUQgPv>p$X;noineiLyyKr;~-7}Vz)1_5NbyJH9W-Zrm#<7uhi?=U{TVU3P#Clo6
zf722N!oWc_1QB1PCsMegPz7OCosG&$tTab;UPN9?{)?zG8Q!8nXh>*dEz3VV`9Huh
zeqIOvuai-qs*faRO!NCQc(S??_-Mqof;PYWR*Ps5qZQ-ce=_POQRnlYTw%X2$g#Ou
znDl$SB^)`MtuJZlMGOk0!p@WhYHO{&I|lc;b2a_IReVb@H&h_BwEoJs!4p~<knm%V
zm}pp%|24t6M|4~hkUhk%7DgWy;%mx2FdaTeQJ>mC7j6ehytIF`N^8hMbRyy1wtK$q
zf2Wy6eo_4gEqIQ%m@u%&+_l!V;|}Y*IcDBlLyX{b5c4OsVChRXSp)sd<;}9fd%Ui|
zn2k<jY`2y4DR9(;QK7ztYe4I`OPXXsi6<7S=q`>kS<N1##Ya^Qn=(UPbhLLhAAIjV
z<;3xw(5Z7|h}*8lPqtbvtff7gcxdz1{3t7yhvAaxf2Gud+a+w&@t#@cQpxZ5S2!Le
zZIghbcTO-nRVXE?&8OLVg!kEMUGuBqKV9iBW!Sr>(&RwEgoQLbC<K`sE`?k0kx6RS
z?-<t!jH_^1^$*k(e%$1fuKPedR^hu?o{$(B3LhkP{e@^uzcx_#)tm&aFhnm+c}Rpr
z)waLd;;XbUtfe_$S_m!S56~SizpP%jpAsbi?nuH@)?*acPXF<Db`4go3}eeM#(EF7
zSD@MFrtl1cHA3te2{7X@p;_)X%m}H5ad^Ee$6`30J`$n<cF_;4tuFnx68xVD|0u_4
zm7~At0*I>KE)v@{#NmL%omASJXrlyEPN;<wQjJ^ext6KdQ{A9&$2vmRe{ONHJhLng
z>Ar!FV(=*0TGcDk4hrcNeJ&;xo+g16@P)cRKLahrQ-#?Pi)0XNvq?eL8-UO5{5Q28
zKm3^4$Gz7&t)^Cap`KL6Iz!Vt@VCA@rLLdWgFxqGB73-}?vS=2G7g$r*!_2na%EeS
zUlode?!;CHkwuYpY*LH*4eV2`PDBOHhAk5=f0|X?QN<>A8YvfR@d)!=+H5GfwL7GW
zA9W|ji<CCABtSo(9ue82&YTH3Yn7Tk*S%jRoZ<?VT)^dOwtZoE<xAFK1`I^tm^kye
zPvP)?q%4|@qr3O7c2;7W@*2C^Rv`~ko6Iu80M38+psZ{mTV8?s5!Cyel(&#*;gX-=
zJehMZs@%w2_|53AsO^>G;Pyq|O6l|az2td>kBa>H*-(k|+0rl7)XsW(UY2n+zTv7A
zl%LK<dOmy`CvrnZ3cjz-lJGNpOT2Hez%Rv;{)qnZ8=GqVF2|f2+n&qU9gY!1jEV0=
zf4jP3BqK#AI9MxCfzeTh$Y6?Z)3?@eeHPX6*Mqz?4jmT!i!1DtL!OY7o4vW83~?<$
zbi7ak70QGfx-NGu34#I{1d%Xz^*MF_yAhiV6+CWUgnd|5ttv@uf_rh2wEp0GeRDLk
zc5sLEp#(XU;pf?h=Zn}&v^cV-1?cDCL86);-NlYhZ@u!EYpDaF3N|QG86$3+F^+6M
z<g6Mw&2Rgg2`qBoQD{gH@xz~svZKM{EA=!5tgns9<!m#6ufhMw&C)i4KK;hcfXJrN
z?p^vTnW!;VC+ZGU0ly7Xx)(}F(sNTAY~R7<?t#WZlR?0$gP<orR&y#p0UOwe@LvOG
zSk>o*c;rDaeGL;&r_i8+dbzi5#0@!cEyfc;w$aX7Fb>~mB!Yw3ADiLgOMhJZCQa<A
z%zZ)N6KpX*-)HjKi9ZD#NQv%W7te&vD-GHFLM?E_@dylo-?ZRJfay};<uRbvmr@&Z
zNM?^!rm5Cl^T`0RluBaozFlIo_qkvvRogj1iqA4e;KFJK7>PlcT?Aa<7Isg_gfJ)$
z1pSb@r`MEWs4KDmqswPQV)qw)Yc>CDVFls4WTw$SNAFBjl-=hmOG7nr!c0qw(w;m+
zW~PKG;qN%;xGl?>6`ufteGn@=1Z#TD`yJ@2K3`G%`GiB?p~IK#jKpQxxaF)l>4_S>
z7`La!{YRT&&f$2Fb39uQpKC@wU?a7DmF=+J@<mOA>5KMIH?PV-f7l17gShpt;&Xat
z8!A>liHER@c?txj;StPAN+@`PljZT{wmR=yf9kke?Irm*FY-l{gk<fwIj!q{uX+XU
zYSw1tH?3s(FP+X`cAoiMocsL=?Kc(4boyDN#hn<OyX(z?N1gJUY7vRXd4v{@)4w%7
zKDw||s*_7cxG%5GjHj6?Svk~O-Ygi|6uBZ`)p$?e*YU)3p#@S#vfm*tJwA6gGDWll
zdnNHyBS)=L#3=S<G3N^yZEQt=o7IPm>j(69RYptLZl=gcFspZLOZ$Gzii4R>f+_=1
z=2dRhO*}Q_OwIZSwZ?mN4BL{K!Pc<=_20Ke-<fD8>Z^WDLgMpo!)96g$6=kkbC8nU
ze>RA?Yi>ZD_hV@Q>^52>8aS0jD&5uiE%9q^sBTbMH67C$Rkr2YVY%^#MmZwW4|Zf#
znagK1Y-a>xSl5H9y-U(ClgL=~f7O^6RyL}Wct92XQIwliEokRL;&-1<R3jMAp5E>E
zlVVS`rCWJ-mwU6fTVd^V-!Un#spX05=DW@N7xI(z9S(w!$lt#1@6>p%KE&H_CJE9n
z@Wjzp;nt7L7m*GHiRQnddL1tQa24H`a@ZZtZI<niqEB;V+-SBAFTyWYdrwZjUAwxq
z-E-tL^Aw9Kcl_-{qG$o?7*=gFpYg0*ZfKo7=KZAY2_qpn@oC{=32CJHk~GX(ZTeeZ
z=~E|qx4Pal$lve$==au;w>LHtX??GPePb)^M-R?A3&&t8Lj3In?cXthKbee}WhO9o
zaYjg)rP7_#{d$?_l*&`SGXzTXuIUL%SMP@*x6@L1PER&_nY16p7x{ScofP)VX|{U=
zb(&+K(YJW&+}+n2ECz@+B5y}0n0iaWgA<XlCS5y351AQsLc1XCW=g{y?e3zl#4RJF
zbrbWaB0~*4{ah}%l~~=wtYM(HfJU5FhL2rFZMGbiHP(g4;8K(sm4$cZay0(Op)@bg
z>Dal#jeFF8#@DbSt|%U~4ms|&*M&>FnFz8`v<RQ-+C+0?y})C&8})01+*@dXW3HWx
zgGm0ca>wscl?K%#Db|e_Sk$K(&*}>1tyzO7{wIR9ya`h5h=A>qDc?5>-3Rl}Zl2$g
zcDhnNe&^aRI3zNS_L<8|(05Qq?&oE;yW|WGcKYRa;bHG<Pk%7RBMoVM96Mh@nDKL-
z_KRa5J|C4F?Ytha+g&fUcSoo@>{a8u8+UTUG%>P&?~X@ua^)H)loRs1X({>)aN)Y|
zn;#Pj$SGz@TB($a%&SVnpW*~jL~}RzG&?)Oz@9_Pcz$*1qx!Lw4J|d$Y0*Fao;dnT
z-}N(*BYljEu{4C}XTaJLQso`=h=3N=ZZX&3Fn86b%4PsE6@&cA4#K3X6diE^o?P|v
zOzv8kIhu!ec(M1AbMbb1lyk+UR(`ZNC$sjkIiXjGR<Ot6-lYfx{HerJc!?`XX8Bg^
z1km`rOix<CgKbKvt57$1?`Kw}stoeg4q7@naxknaaP2wQ>uz<Q9uZsBa7xdy8vPy@
zujsV2)CXz3Ak^cg^l=aSq&j?3a&e-kq3^0s=VwN?rLX3AICc~-=Pdu1a7^2wGEgvd
z_c$@tOawFzoJP5udfjQiS+3Z0r&(Pgo|@1iH4zLbpgeoIqLXlLrT0#&3v?&ok4*|N
zRO~+cXP9Ub8W#jw*#5`mQ(^WS99k5t0*tPoq7$0F_}E%$Yx%L&&n9X&b;~o+u`5#C
z%1sV=VabrqDe$Scyv(RL>Iz$9fr`68fMeS!b9iez;SNpL0B|fK>5}m7bz1D^^AM+A
z<nHE%N+5HlZ2;&I(wq<@Z_6F;t?VMPWcw}%f3EhZRb!X}O+%jI?m~?sQ=O!1(X=BK
z^nX?EhE9A+p%=45EQ_BAcT!GvGKxCb)Hrqs>|^4qn(aCu4unykU)wZ?9+#Uk-~V}J
zzd0{W+1<b5&T&WUyW7(xNynJ=SjPI3YToAPtu(;Vy@pt<WxsXLri~nFBa^18@vLo*
zu4=EMd$pwOjq1ZXVQvM&9E4!LuS%O&a;O9iSiT=<25T{HT-SqwFR9qiO18;F<*JZu
zU7($){s>zyUh)(K$W(fq6#cpGVu|CegSTI^``MK~w}ZoRt=jAiJ%;)y7@9sOb2$+D
zbLIEei2nNhs@pAceW3&0>a}ZM@A>Mde@#4kcLsHzlh<Zb9j0bdvw>Q4hCXo{{@x8;
zCokGfPpq&03$5zs&-bERKR?+x>dlXz>?u1NZn_K+=cirDaEO42faT*Gt(wD1$V2s<
zEz|~WkF8|Ip2>?X5FmX0ydTrey_%T$Si9{g=-K>?16>~s|2M4tVu<IfL2cs|*p8?D
z;Fv%jyBZDcyppMn5r6;(3-!|wMiZBZ7~HN^>G{ACQ9o!tOH6$3p!~V7T(`eCqi>`t
z3j`wk3xtu!fi!|z{w5k}RyF-n@QEolEvW+9+LAu%b9c3}DhYEEF&uK8{$dZZz(qsy
zy1^}MZj`i2^LwM$qvthM!1nzhMF60I6IF!&)atsA0Lxvlz5oh0CF_~S`u4iu?L}Lg
zcR)t5ID#v~hy=G4F6?(Nf2Sd=jv-<)c=-XrCA^Pl{H1mio%ThS8wgU^ky4_V)-vOz
z<*Oo?Xbg`JDaD}zM6T2Ys98kW84djj>_3PEz$JzThLS}Wm*GFQ2^Y*j>``765V{<y
zyb<aVbv{}>v<@>xc4MU8BL=~M9K|ZYEB!o^6Du~oE;u5+cd7@eu<8YpbR})E(Y0mg
z{jRXH*<!<g_Fp2KkuBlYr2aaD63zL{E!Z#SZ7-YQ5*T#mqx40IBz%zyfH0K_h1cM8
z*h?S;DGLh^ThZ_%;===J`e!V3>udXt*U!Jqh+;4#+}z90)-3t4vk8&5Stuv51`FeX
zrs+x186J;Von~u@1`c_uG-xzSE2HA0U6z*n=(I3BM+gMWMO^O|V-B8wt<+bsuK4+G
zD|BP{U`9iy8y=W0Aijc$I_V2^rpsA;zI%bB>)wR0jaGM>8aSxYNhaP_d;Igj8T{n%
z9z@1~47U{tW9w4&OuJy5e^pMH#|6^U3Cc~o#q9zBcu+SjaVyBB{$xPDvtXfFO+^o^
zl<V#IsuzeGTUi?WME`Nf9G3$EFc1jp?d(CP+5dP9=)>QnFAk?z<W_?Nbr61^_R_1W
zPNKj?Do=Mhp`<Vn|8hkuwKVw5V*x=#o3&t`NlSbc79en@2ZZv#NoP(cqCVr=ckQ}C
zZj41?g5$1U$UDuL4=DX2gy1xj)F|4gw`ZS&QC5Pa_4bva#Qj>lzOp9k-8F_Nc08V8
z{)&^iDfHTGL9cdZoP1R}q@L<*o&q0{-Hx<?k&X!YNMR=Lt*hh92jQQH>K1?Hv~KiC
zV8))EAUSxZ4y}J;-9EaR^<SzvTpeu*=0SIOS&+wJ?>jkzrZ05n41IWsX_)|c7%&C$
z)k$f8Lp#ol30~btLtVn_F`_3~QooiIRoI)~F9U+_HHA>iAY(~a^5}^kLrGWkboK?=
z0t{<xx#kkjE6cUiv;BiFIMLf@)x8V@w7xYd*=*Z*FFP^PNmny~sBC1oU>I`XT8w2b
zfAHJ^>tWfemFu>=@d#tm1AApAm+&s<!JPqnTduE7aWx1i3FvvbH_go9yb`sf+7_)V
z{V*50z;W{!^!I~}<ZLYn1OUMGO2D-I${`q}NkYa9-?5y1z2^($AVM?0+b%*vn=YUb
zbrw6a8rDXDjm8LMz}m)uY~)C$F-GJDf7WqWS)3qrMlKqN7s<p#K!6ni(EvcDjBku(
zdnH8y#dGyStcRCJD0u<sz!})zEu!DSUnlq+nYoZ9kY-H6$kbV>L_qu)Gy#6>KAVy=
z9zsT>wkBaw>br;y*hBI$p;##e!4)xLQ0syKYY8yU3F~Bh1C_ls%zbafzg^tS2z|&K
z6UWuTNb<hQWJR#}7PeGTBP>`L{1544Yhg7z?NB%E<tyMLMiL>KlFT3t;4u<r;}?dK
z)6N^+EwZH~a;fo_t*4n`r5deEJ?g%8v4e;(bS!6bxM3=tYu@VK<;5*zd3O_67*)b^
zG77G>y!;p=AoF*aVe`4r4)q#a#}0>?fO5~Z%ux9#B)H%}P{IiikOttgAdX4UXQS~_
zDRr$lk%^}HE8$jKm2VV@?RQYx((!>Gg|3e!|4Rhk3zYL&Sad@CYny=kF84%MbI%nj
zjdnRGrAj+>Wo;$udT-}dfN6I;&$3+m%CvHyTFpXr>*Uj1&V#GkK^mihY99Ed-9c+M
zH^=9wH^{FXkcNQ)U8;9-V{)I$cb`P!YH?v{NDBCPg||MS;#?jLF=$#$n7kdpa$_S~
zD%Q8j>qBoO74sOmR~uiLwe-ZWRR38ZdrX2cPpFJ{2#v&rfV$Nr`NQo|hZR;1^$TZy
zJEu@Kb8J^GM_y|6d+k?Fzw0vj%rV-ecE7Qy+ta^^w^!Nd75ZTfO{P~K`tGpnUJhKO
zZRf%%KEZ)ufF-*hrY0`4>M%fH5+O`N39<<22gY=V1B)csBfjq7Oy#~uU_{Y6H<VZm
zFGE#gc_(#;LeqFCl$pHcm(;f%5>_Acb;Q;Pv2b)MijR*67XgRgVf(>n4=0C1&4lF=
ztNWE3Z=D|x*NhIzt?w$)L_fuzl|-k_KSBGImAfvhvb!7~RPwX`1~oV+d4+KFu^^zH
zZ$E9$Iz*tUjb34W#qbaiVEq<I*3G@J3w?(blsI0Uwu>xnWIQy?hju8S%=gI*khw3=
zet-4)3C*rv6E_d<Jws?p^@z2EX7Bff^*D#MM%itXE2LGc`MXmT0Im%Ow*3qY$?-{7
z)R(qT7v#oo0sTcs$X_2YTK8173fc<1e4ympy7}z0X?*~A)d1ziUf+CzI|%JVeB=)|
zz)L?2Z@*0a#`=P*b?-_HO1vXjfF7fTVv64foxf&y5-Tf+G`{3<y(x5hKjbeaZxnKD
z)=r5!i%62TvUkLClOhTTIR2d7Mf0)%P8ioP?!H0%Qc^uPP7!63UtUv7qJZ%MZe$An
z59AWkDv*yho<ki1x>m^-7tf&N!&%qUVHOEbmN=xjL6!D6JC#4Aa=46T7-Ett2Yh`0
zmCyXSTm9vp-!BglaaOt5b1M3>j~N%`3<B)pAlTmTuCdNbXHXLzibIWpFmN~=W}8@)
z5Ltn#Z}Lw$Pyy9`T#UhCRPN4USZsbwLe0D@?giADDC`Iehp8XmLZL<flq)I75*6Oa
z)B6M3?k?79CkxZ+Jr_Pof6{<6n)B2^j8dolm+_?N$4MxEZfR<3Q@%%8rvgd9UBT=R
zQFthmYp1%U-B)S4Z$vg8PLMQ(lYl}Q2c^>coRyh|*l6(OT>Mu!{<By7IMeo)i_(YE
z__8pY{R$f+`S)3;DNo>+iMa0(oT@wfvEU=FvBNk+1MuBF_UPnew~+y1cYQ9@3@66H
zbi?>Gaf<U2Z!rta?1uNNC9!;h3l{DH_Mm6vjN23!kcRQV72BJB`LCqoqe<*YUi<<?
zUrCSCzKAU#yClk1Ha{&$?3qs<**Fshw(Sox(b%X4O&R07xkCNS3<`nvuo^kBR_P}t
zXzWX}P`CV>d`+-evorG;eTZ3VI(x~{{zsgLtFm|}f0n%McGfh1(Uj>xw5qPRWB(NX
z(AR@tVGzc!^usA4cY;}RS2j6_Vv&c@S>rzwM+=wDNIt+nKzYn}32UBg6N*fQxbzl5
zbqf&`0A%VF9okvOKJW`H^8J#KvNTw^dvLaHb=+E_mxw9m9aG@JKyb!@oB1}?L^~?s
zg7#1m)h7B=NCdwdiYMApfw6`3%Soa5u)AUPQ)>N#cPR?=*+2DV>QAEU!|zyoF(y)o
zcpasAB9AUM*M4R2O1OUDu2INh7h8tkT&t|;N+*J1Twpw)c$tk}htQWOT>SwCmb8sH
zkWvB>iZIt0k)zU^mIL&wuHc&N<`v2fNdw)xd{tX27`T(DL(M8f2`e8yj&AZS9uWb~
zkU*552Q0JWm`yBwLAHM};ng2mxNAC7Z%dtt35PN8{*%uhRs_Zc8a1KihK1sSK+s=T
ztx13To95jCSp0BTGJrS=+(>pL`Nrb1EuXL98~XF7!b%4@h!M*|(Lw!e7pu~;8l|G+
zosHAa>=;D%nlhy~k+5j2Nek*K><A0&E7z)pa#3bd?tt@-hVKF%=R`K&n97F%x;hD)
zXL{~5ADNb>$yP*(<G}eD>>$~G=rKLbs_t#<b6xGcJccrHB2f7)ymGPB))tzRysHfZ
zifOY_WDC0jUhxd(=Z?q9K)@{ky?kWV`g|16=(9ay6qp?90Z2nMwB`7Mg{j~O(|0Y-
zNnQdY>4OCW(@Lx-qm5IywUnTm@I)J-bGUPws}F_lptT4Xn=ix=VlAIaY__z!^-owc
zl9MOxbS_~<ny4Zx&>hEzGi}6%MPExb$s&QBOP~cncS!=UG;0dkyKE^}uo`>^`~<aq
zMxK11mP5z*9%vvsF5c(TY1&B2#5MZ@wTDoEtYa*&6}`Wt36>is{~NqZR#OrQl^M?%
z;U5O>BK#j1ZomGCsmd5F(#7y5PsV^^s&GC=U9v$#J_3Pj^s|eIvjm|zM%S#oy_m3b
zmee?=B_ThHPGou+EhYv+hZG21V(PA|<qrxK+@uU-f@-DQSP=V&&le*yD(RhX>*cl~
zhCine$Gi<h{U2BpVl3%VLiU5!xUZzs8|jkT89!4|y8scOEYk$$-%bIJd9yirK;vY@
zbh{DVLu%g=o&Vx1W3@1zEoMdhA{T`0<TcTNn*HrM8R+l`us49}olBGLY-8fu1f~p=
z)>xz`kx>|^%=Q=oic8gQecr6~SMG%16rta?z!^Ktcn^LXGV+e3KsYt+$Cru6bg59O
zTu?=$>iRmnPkHm-D=`5}hZ9wNyi}j$5|zb!Xh$^Qo2C2+qu^6nkl;w8g(aJQtDr1L
z?7a4{+On^`tf;C%wYv{1=m)#cA3YOZcl>(|1pVYSxu>fT1}Cz;Dl$O2NRQ&G-2g>x
z8-96Tjr?6wGuKVJW!UOZ^A6~m{1t%X%n~@VrV+UyOw1IK>Y*HICRHq6r&7cf2@q!u
zWEK(LrU@Sjxwc53!McN6G-G4L_|Lv00*JUZx6$QPANS#^MZKdKWzOeSl+cf6#{NPh
z91+l{AfI|T*-jlXY$&iH(Xa51K`t6P9MBrDblQSpm*SvW9xFe%Lko8<W1r~qxV3R&
zRitVN95x5Yw_@8j(WOoBEGj&FlIR{x7R)-KV#LHjaK?ortW2q-93dgw&1k1=(<-8m
ziA-o4Wk-Mw068X3cpKhJ_0L*>Nnx05TIQ1Pn45^BV<drfpM%ufW5czV^P8~s5!RCQ
zQ6@BtF^Hyrb8`KhlJQ2@fT9<=?IOnyTke8qUwUG#<|H?&oA|#Uak%Iv{u4+r=<-rS
zYHk`YHm1)K!I`2>%LnhsMQIi~8!te1NfUn&xq~h%xd$Md#>pG73=V5&byoUsSCC;R
zZ8@n(AuW%Zp^IivLf=wpMoXt)Dy<~3T|gNYbgmtvCjGx9+0q1D_Y#~b_XTrXz?Uw|
zjcG@cG;o}mMgkBtvhqlAJh)P6NT!STt@7Is^=^!N#DpWy*dE5OzWyDH%_XMV<O#~0
zTkI0c?bj6F6i`m6X+v-kz<ktxp2Cii@(A+D`q+mX_9p|(($&Mm{}_j>$H^Ttb0*He
z4UI?dS4odZi-8Knb2>Q**T-&>I9O(hA>-7opoFBvT=y~xzsR*AyUblUIQKdoS$A(L
z=dFYcBsd<j1WC>pgKgVXaz#?rp4`)^Aayz&pTwMR3b)jQvrw;x8o?8n>KH;`PN7zk
zcZn+8$Wwbk46P7eayKAr0hjG)ivx%K081o0=|8KSH#_=W2006{0^&Qo_3aEX?a-Zs
z3YQ<amS(#^#;9;oJhHN@p+7S}^NbSkhs;K0+^w4>RavM${v-cKYZRByuX)4{1}q)H
zqgCK0sK<Apc5#7kV)Em3#*RIbkbi#Oww{O_9R`m@z#cIL6DP_`00Hc9gtIi2D$HVG
zBhU5|<XnwiJ`&CvNs;9-?VjoF)%cZ7WPnh9IG6I_ryfM4*15_ADGHPi;MP@~57y}#
za*OLg?3|g%msO&2!4eXvK3u}U5>CWC@wG5~Zue9kdoFC~(hIJDekfg1^vtp;VcGNm
z+{7cZu@G5mK3OaPE|<)>jD<+@0y2&gn5%gY@eN<+9D=qLl8X>NHo``(S;%>LtvkMR
zi*HB;5UlI?U&!<R7ZWuCik*noFpwy1gn%R@B(9BQI-3)MoUMEEH5Z1-`mT1-P+My9
z<NTXt6N`NW85d&Er8o?&&r97G@17m%w-gnyfEbmNw%^N!@1F$=WY1lAPiq(Drx$J$
z@g#@-v=yj^8pFa>*b-0Y?ELMr5YbCzce>ltTkzPNW^e9#5qD-01h--#8MF43i3EsU
zqq0>n0JwqD1u^BRL=szULn$!CO|h*8yYKoku&1oEzj%Ep*Kz)ka4ZnST3no^qMU|H
z!)xu6145DD8VhMF1q<^-XFcb~TB|R*_&?8`v`?fN#<-nXwo<98590cEMqT~qcbSCt
ze0U??tdMsO=0CKHn5x|)Z30{~_d$HJ)&&a<OUOhh+q|dW9$p~1?pd!IT~szj#CSY7
z|0Sp}vo!t%^EBz|Pn`LKHyWtS{T|R{l@>hmINF{`SE4wV#`lq%mwmTryBLoY@BR)u
zaf2KK&Hmn-zN}glK%G-O8}f^7M82PjpSLb|Z_S!!vX#kx;^{W4Ht!QNdHwou;U_Rz
zcFlh`B^QSXdjedVyoY?L8kn~yJVwA(OTAl<{&=zVY+pg8C8Yn7VJAV$`^wYxp)boQ
zKq(F}2q*>$8?pO5P^ySp?2@5_(n9>kHzSEZ*5<3y=;rLy=Em~G<XVZ3I~24brskfv
z_9bg5mF3jhG-qj=s3b-f2%tu=9pPUo-l0_TTS!%)>9^W#=<>O3;y2uUm2_@4Sykgo
zu~;3f_eINE=U;Jq!a@**32L-`ct$fAeou8SL<xdzBj`&d5701KZE!esuRO1mhjiPw
zPVr}{JYtwn5q&UbUS{~BfS;pbZQRZ(gJCX=$IpdtIt=PL;ga%bm=i@qZUJExQh7$l
zR9{df+&KF^@QuueL)hC@@N;y`7IJFcc7<k)!GDJ>K^W3olb$BKKm&=%8jA_LC2qNM
zK`++(>j5~XtH=612BsLTZVu*y;ksohj&fxR(05*lyUkI=>^bY!9?2QwzoJ+U)deXz
zjMPCggrFkZYqmVr*#i=$yMtrLxs_i&9~!#&q=pvn1dJ)PgnZlm%>8+_iaAb6o_GNc
zUax*oX`q)hX2Oi%kq4zVbe{1>ucCRn9M^0TbHwnSPvRTNe)NSYm@UX@SsfNUk(b=M
z+A8eifM1t=8Ra(*VA{wmEX+7v!F-a+$*b>d$oQoy35*0GXv3O>&@RKauXm9~6&{3#
z<kZwYjy?Nhi$pIx<R|Fwqy@@25cBFa@vmN{9x%Dm-ba1I{ODsvMJehZYdq3MhcLTz
z`<G^!?J*u#(&x_wmJ82-W*u$!lN7}E{Y)SEjkMe<-n7A{RU>XKgLQ)3=j+Dt6A!i8
zO3K~cPBf*|wQsCTh?kh5e}zozs+`2~qu1(b%2kqC<LdA6UO;Mg1S;IAgCl3iyU{Am
zCqCA0iRDvTmYE~ktAXP@)GV1ttNezp7bI;fO(<6VFKGM!^Z}v=J_eW!CQN?~WNL(g
O{$wN+#UY}`!T%4CI%78g

literal 0
HcmV?d00001

diff --git a/docs/img/versioned/learn/documentation/hadoop/unified_batch_streaming.png b/docs/img/versioned/learn/documentation/hadoop/unified_batch_streaming.png
new file mode 100644
index 0000000000000000000000000000000000000000..0fd28992a6b887219267a2565abb0bf08a072b43
GIT binary patch
literal 17371
zcmcJ%byQqI(<dAV2^Jhea1RzFID<=o;O-KF%V0COdkA3&gao%>3GVJLfx+EvaCiNN
zkmuR=J7@pfUCuqj<xbzO>Z<Ch>h50=rmQH9fkuq><jE6^_cD^IPo5x5!0)$Ep2I&6
z2(cc1A%Ik+#h(-okZ!{7b*$92!P*M)026y4tFfv5M>AG8paZ<slP7|10QfD?3~Ws4
z2DG&U0o;VB|F!_Y?;jqsQB(eH0=5yN)>cral(2U)qvU4gVr8cmMx&&p6m&8*2dGL)
z{d*n$mk_lj80-LGV{>(NWp(9bwRf^$d&|$y&&JNd#=*e?Z@~g`w*woyvDkrV9y|F@
zKayr36DKPNu$8?X<wL*5AMKsNLe$g`g#P>U$S2s!{C`oh1N|Eoe1L2Z->|)9WoP?u
z-|$xjAD#k~oUF{?lpp#Rek=I5<^S*5KRSYJ55)fmGmj(v`xHJ@VKhOu|B6i*O}>?0
z=gAY%C+{W2)ZGwvlQF#T$7(vEwung2Fo>zFW8MY&r3L)_89~gXiOr<kRQ%Jj!g@^a
zWZKZihV5ir(*`(ZomZh-JblHYtWs_nDEkZ_g9$~JT_xkEY8W<ckSR<U+LHn<o$~VC
zwM#ia_v8dME*w)#x(>v1T6~H6<cIQq{}P4a$=FDJ)9D7`bK&V?76i#kiXuH;x?+@s
zW@u?XqE#z5E|!2%)MVBbT$LkZ{9OwXT53wfhih;R$uRgCSl%Wblol!ODo%8c#}J$`
z4Nd<dCrR`)<8wOJ>z|IL@Bj9g=84lS=DNxLo5;V=2rt5unK+gZgVX5MG<~ih>=O@3
zuV4!xggOb!OjnS|zuRlW(z2Lz8(m`R`N}3=`~)g+Aar<1z_^ipv|_E56{?&$Bs@;U
z{N@|hS`^JGdkj$KALD5nNE^0%&*XF5>0=;}<)m+kFI_Mv$t)<bz8gcpRZ`H+n9aeL
z@7h@JQ?Xr~hr$RtGwr3~1|m7od!3R3?$|vB4sMkpnRCAqx_Xplg=5^&(qg<A+R9lh
z(PxOBI^*gsu*iI<`4}rEjf0Kwa9Lvbr)FT2eO3cd0!qDp;laRh7B9iH;OC>?8Jqc&
zwOL~@BMz0s<1KW!<h$hJpPB(?J^&(LlnI)>j4=i%O}n_CBsr#vviT4AbJV|-?Fr@3
zf0DH-L6i*;F#H@sxFW-k(+jSYpzrh05n1`1j{6_7c>ZO|2?L>!kb#cPH%V%(%7k6)
zX3t|BDa7Y~#bju90z;in<d*Q6SCVK<4FML}Y6e0bd^vvtI7E_ve$#Snt&Sf;0XT+O
zvC&cVIv~VEW0|=zk4`x{aTUIj?oC!+zvO<ZLWSH}Bgks_pP~Merc`Ikwrg)gAxEA=
z2Ry6AmaW0;wMdKTVWQQ1$N3_Hd{cB&elv5UoRKwRV2^8qgCTiA-%@Z2|I?eulp+YZ
zf#<+)<kE3|GctmqI5KC|JCn$~Z#_jFt18O>td5v=@GD*|Hl4Y|gqTvm4>3Z~)Bq>-
zZl{{Tv<!)22+~LrO$KL@0sVw8;;{n&^9nm=;98AAI=eq=Yu(h^oc!#2v<9fy7la4{
z5d2jSw!aiA7nhSkpemRF?85nzJ9DKQ7KsnR4&Tu^zH`8r>FOO~mG(auTkuy5;DEGN
zGTcAQ!lP&vCG3v2OpINYd0xJsV3~yZAB(5;4@uf~it(b<^<af~&3EH$lPUK`C_@hT
z09hGX2Sm)$^@l+;6H`M-ipU~pYh@!N>ELO6z=z(C8-as$ul!6$Ndde24VGavXZj4+
z?ZNsq|4XGW5nz+cW{ppx<kG=@U@m+>b5|OvyJ~N`mS|DXa1V!kP92ggGC|A&rKp7$
z4wx&3W`RP~z`$ZLDg2X^Xj$4q(6A2ZM25wm)&KdnK`?tTP?af7&JCBqLXLd_HxmdZ
zasa#QlU_H&{%A>rifc)o5|PsV8yF>G0i^?GuY>9Bv8Ya!uA2&_V$OfR>ZfJLz2ti~
z@cTfVA*ZBDx(!_aA3MbM*Tm+-G2`<vKwq<wAT8-FK@5=mr_BHQlK=fp8m50aAidUK
z2c=P&w|B4K^VM=Sf}}}nslDi>wY-Jws|baV(%m23@}}mgA9s`es~HEqEXl#X8!+X@
z!V%IQ$?Qo$B2(8y#)a;1)}Xy8ks;RiM;Dp-1VKcRVuK6T9u^t#amvBgls@QqNJ@-(
zno%m=A;~G<h8+Cz#D#&kaS|;n11MX~VC|?DznobnRH)oYDx0I!bw&T{aTJu#P%a#=
zMCm!{iJ=kXSE6O<CTT><^lwN)AP{oLUo^%+ANz`hAty;lZL^XAtqNq-J(BtCDG69C
ze$Ru8#+6Nif&3(&9myg6e(c)`ICx4qT{omEfOPLm<!O0iV%~J^=X4bGC71g30vttH
z7^N?ySM=vlrWLp?1ug}QeSawi7+3ZIq$QH)-t++i&ZsPp-_F0aNP1*0Eg9uNT3FMZ
z>Z2l8v^1pHFnBo33&mMv^o<^hH-2x_L{U2aUGV~zH+pZmQkuIm+IFHD)cOvot>e?j
z8bk&te?aB~?~^a3!vs07f-9@tvWKK#Uaal>yIB}t?n#M5I?*1O?&0B;*z|K7m;h^c
z1uBno;`LTu5@Q|DAx89fxvhs207SD5o?-w-J_tiT_B0<Rk}4zx(~P}m|8r9w<(-0H
zMwBvQZb+d8nP}}eCcY(R!g=(Z(uh7uwHGvkaM5OfoWA^gS{Fo*{lghXO6851#hHL$
z@I0$5J?VuRaa+Fq%hC7j`d_-`exicE&ZLv-BtOF#m$p!Tk~N6>EV+$kh<(@i5=R+L
z8YIm1B7$I?n}!E>Y1vXRc_>K~kUX9@#NJ0B30+ysIQ~q(+79rtq&j)da4l<m`=(1+
zInI?cW=Th-qeT`@;uXM_-c&J>VYTPX?M3HtLh&u$zZvTP65Bzs@Y%a9Qv)KD@5@Ov
zJoLU1DKRS-q$f7YigT<%nqrJlaeiIUK%gT=&+$+Y2Pv<(m8WowbKTX2jvzp;mu~~G
zLHeB1m|=z()$DRi8SPqhMvrM{LY1M<5V1dac*;!7gf5QlOS|AN(AN_8Sej)S2A$y&
zg}~v<h=9^rafX>}b~fl#PPXYgE{RuNU<TmNuk<q{2xf_dQUu60G&A5GBKm*A4S3>f
zO7cyf_NZ$DwT*yvu$6}FgW4{)CS70Gw+~+VK(M3@VsIQeG~@%B|Bj>5reCYF4}cA;
zcj($c8BnV+xSd{t`$yWMk(ijxNO;*D?+Bg2{)t<lxHFB+^t`^_W_>bVr&UJlc5khF
z!9M!xs{4D&e6_#9$=@6sg*)1UY?zN4)Jn0)I7r9(Gi;@22z=Ka8R)bsiOhxCOW}l;
zjG2WmE8Wb)AWs2=65p30^{k{9HZJ17PS(Z*zgLfJcBBRq#9(p|$}YYA^UWM(w+`>>
zMLAOM6?uRmx;JU>*)r;G0+#dTq>mt@#H#aPl)vP9mYrX;QE?Lu_CP>kfNH=lY)o?O
zT8}uw(aYVN4=8^ZgSHX<YoOf`MV<OFfk8wZVC-N%GLReyR1CEXIO@81%aAMsT2(bo
zK+MXBL(ekNMinBfeNl@E!1NF<N4sJN<HSH{U$6x7(3P7JpU`bcF_6&#vj(FM`43PS
zwsXkv(!kVOqUdQ*F`@-+Jzg4bo|x#T$yg132q-xfJ)cfT?2dNugpLem-YP4~Eufn<
z&ZPg5cD&Rc?_o;1c_J&EmrgK<Ggy}fS-?*MCB$v6y5humQ=>a|2MsG8eWJJBS7q4M
zK_o+Mc&dwJX7I%qGXdS~noabdAK#UseCizGzb?@>m}W}VOyNY|;_xbJb5L+Y*)o67
zcvWKaLqK$dsBnz*`0f_8g?}4_OF|xjH2vOFXB@SUJ;jBcg(07YqL=66wnPlUI02?*
ze#4*Dt#r<xLFogUDP=`5okwG<7f_?=807xY=huO01YMDT0$Uw~V}L{BlueU&b@c;w
zrd7+!(sd|g@BWia{_g|8f4(KAUF44Y8B~q5PB75-r~6koS9&=q-tS7NCQV9}XZQK+
zv4Y;f`3)2FT*T>nkoaBamU8AzOv{LzK@#LKM*V&-I>5bNE0qkBF{!7*d82srCDukz
zWa5JS<<*5g!>G$)F<^=}QQ0-h{)W0-1JOc1Mw3jJ)8`@{MedQe=y=g8*FtE%J8cz!
zg=0waW+(;bXy&-HH8s%C=un@s&k}hN^_^<rP0z)f>);ZTE=}h`9BYNg)RqOQHVagN
z$&Fuwoh(9jP&*#a<9*uey^v}zK5v2-s<?6nd5o9-Nc4jRN!zVe6vQ}a@u?=VF0uhe
zu_1q$Th)(F@j{ZFjU<Th*&a>v6sa}>WKe##MC8`!j`elsos2<bBgRWNbD#aK!5Wj$
zRGs+{U3K+G4gyhv0Kevga?gG{V?jbplF>7w7fPbvNX^ekM3K1tbGR~sh^I+5VuwkS
zz$0Af)=S;hcRLoNy4p-FWu7ij7!t4QdKi3_A$m@$46V_$h4RsVS3VdcUU45$`>j;T
z=Cj+_!Z^Y0Eb~AzI$2Z*w&gi*;3XWtLzZ!?PcX}Ag51Q8XWw?KzVY+wae?sF4{2E_
zP#$*to^R5sQ__dqcydY(cW_Ti!Na<rX39`^HQ`fO(!N#D1OqeO=f5kQauI*Gfw4+N
zWC3yk<OpAnU5UO3i<jB%QC&*pN4w6u_Tty0-J1)B4X*O-A5-5~GsXp&^a=^sQK;s=
zdz3{6>Zm6H^1!Hl<`6ZMQ;r(P_d9AlL7i80LvEf6mTvi3i$i-5L=?F=FB)nexh_Uh
zuVb01^ga>XTWob&Xf*616%zg+1~eSY-<-Xt-rvH3h?js*Qz^Gr&ypN*++sd}^;V^e
zMOX)J%gfnHetTQJdnYwDs-o7X+nbk`M%h5^X`)CQ{euWi!$jg{P}CN=+XbYe^(rBJ
zy-bjp>d`-A`y<f|s=|nEgmVm~wS!YLMmo&T<G!yMK~H)ojTgP#4bklNJ7}4?NhVB=
zJw(WYo%F#caSd}`pFK(lRTvP@ds>L@hSUF9uOuZZO8Hm~Pg(d|qi>E+75V>|AF1&?
zdSg>+<$aALQx3z*M*)8lrN;dVQfSYkPW)&334Td7sz$;@kB0a46Vj?a-eGR{>&N#q
zUsJ*oMq`yaAB5tELQ9RnKOtGq^lw>5`5P5Hna4*_fBU|ydB^u)-tpM^{Kzq#LG`aP
zC>c?}aUk#8sfvEY!LOj*y|u(=O7V|$aijgRgYk|^yBYp5`atN9mQv+ELbU#vi7yqq
z=d<Hfg0t3W6e{@N=+v%--W(~n-;c`4vGKpLh#fX9(Q|nim#7?ScTHuCHuc*1ZO=tv
z%Z?<+@3(pfOCBpGB32@*Ks1rb-Xca8E~@=T6@G6`ZcRn22iNh{7Kw``$yV!)^G^Re
ztwJuGrs9KJmh+~wrPvpLWlKqi(p{6?ZYF=E=r(`ym4fNb&4<2K+WoLUgbzuKoalhY
zJvMHzD1}#>tIyK)8Yf-{<rLM!3@)IfgiY%@F-^Qh61*R8#p;7@#`S9CJ;)G@376r(
zQog3Hc=P#wDRa7>s+9YKv!wKgQ+qww@s+Ux>FPuJ2ulT3@<U218jUlU$M@c)Wh-vm
z9bLXXW;9H0(QiNg4fMF{)xwv+_;`i5gr=>)qi@oC3$UrzM~pfT;>U%+HQQ<sy-tGf
zEyA@qk7ICo_#?mEdS8)mYJ_CF!2U)qiWN5fiZ%;A?`dff6wv8c&E5kur1r~ZvQ@56
zg`RrK&JJlS37K>f;$IU*tf?gYPZOAH&o(5QDeBnfrNeM09N^6`ny$nnmes2N+lcAG
zP8vAonhTe1uVZ<+JtEwqlI;A%4e$|gWZoh=6*9VreJc>qUOdq?;qmPJ1_HkLI{d(N
zG@NO1!yk60>KYpFGa61C95Oa#;V;3b3@zM3{0QH@esEK8j(OoERvUdv4Iga{0d4}X
z7vDv_e+&ve3h<y%Q<R$jS5r7LqHvn1ZYrkH5AS{wb;5(sg1v3}?Z+sE`xQ<W(GWkL
z_OJW>I*SCbUAgmbJGaA&-!p7fI8fJqbT4wKAM8(Rk4~mTi)oT+iBOU_Y;_y9V_&!s
zzO=^_5NJ@=h3vbt)i^X{SdepSXo!Fhg<R)q!e(CA7XOV_@c9KdxldYjV?eaiINRaj
z@ZS*tO{YbR&7ba$^&(?KzkK2R{}zKY&s}Judirs+S-6U5p{_JoifuQ(#5+K$fF(0~
z2j0d&Ty&s9t~hV#B|w{B^Y$Cuv&b;P2l?K)MzwF!9R6^8w037J0tJkYNLHy^tv9Gf
zKO^slS$W`{uXJ>yx1#>Kix~OyPsy}8Jcc-}BpqWWj*;6G{?72xi?`5|BSAKS9}2X~
z-#uQd0#|O8HNI=<B$RNLTJ6NQ34o1F8^*GAS@dDU_)DHY-!)P&9@Y;^<rhwEcxKNY
z-$v$o+p8tOCJd84fBsVXJ0tr?30O=_%;Viz=j8s^kjKEz;SGhhaCXU?Vzm8U{Wzq^
zI&fWVD@UuNh2FJIL%Q=PEpV=q^Jq`a-urL{D3Hs%yrz|e0_H}zF)GaEiNf&2WH8|O
z+?mWRtfc=w?RysX6DPgATeQA5I(%V~{mR^z9%t`q4#D-z=hbo4E!M+`TQW9|6vjjE
zzy&Wg^<8(8PNe}BYl8yrtW|v*z8I1PBeVKJC;j@h&Gap9A(8nyOI!N8Q1^#OMWrS*
zixn)@+Oyhn2?^n}KcP^`<(r6)wC(1^x!+p4-T@qTJEW4Q+Ra&|czTvi=@eA8P9LS}
znF&NZ@e!%NG}Jw{w-Y)}(?}k-{>-I*uqvSQN-^aLjOV$U_Qf3M=u0dfz!T)>a`Un;
ze10Ty`I>pWTRUbU+I6vj^$L6seih8xJsH<f&Piqky2lYLTaTLED({wr@~SDn#?rMQ
z7MV)kHjB*P9HANBR@#{0>M~W<3}KwBmw=2R484s<QE!>7yTZ6QV2Us$X|jD5Sp=OT
zIvO|<^u=`=zXsaWgJZLtUIN1`ZZ55aJl?z#aKtv}B>7<n(|?O`@a1ikk-{sNv*?f#
zT0=DHN_fyF)^-uz%jC<ljh?D8Jd<&BiIh402sS~wrpED)p;}DL5*TrjzmPx{8IhAR
zypKEI>;6iZ5cYd=v$$#C169wJn*YL*XRgFN5JQ2&l^#}$<4S5lpQsjbi_BZ(U}!ES
zwHA2M#V@RrYy)oB^=ovzlBPyvlxa4av=`k}kN9v8vLb}CU7?aY_e;rh2#no>f34@E
zafZi25&;+UlR<k&OargK89JMm3TPS%vL^ISPTnUr?u%f1zWqiwJF?G>UfEmny6@Sy
z<O0e?1*Vxe(0gXIY}2TY<Ig^rT^>S!SXs#T2_~{|{e5@4ySmdtXIIBUtYi*153Agp
zyb5Tj#CoOa4pvTJ3HAPG({clza8Wsw4>r*4fmF4mp=b`q#EE0It9!v8hK4UacbEAm
zzL&&D3y~i7Q+#W{Dmhak^>9%&%t?vbLLVZ$U2Hd04pxg2bur)Xw)#C;^5g7xEtBf-
zFh@^}Z`aw@k+$bHzZM&QEm}OW!7~EA_tc?py*+L-19J)#O-bs$0bf->_7!h6jOut1
z@+0C#8&F428Z#|H_Qe;p1Uk)K_}}x{W<)COPm{KTs4Q65j^Ed3bBOSOqJzWP35Cs;
zTP*ImdUh0nAfW<G#a;CL`7ydhi~^Fql=PxrS4?+Uzk-!OrB=fWL@zT&g3O=GdC6=I
z1{a(2-@j|{mymtFcHqDAa$;I2`{FjHMFJHddsg-lo&G~L8fZbz!Nj4fJ$>XQZJIYn
z&uG0AEjIO_5X!~WTt=$Gxs!#N^S$eY?bZS_&F*cIe_ia!-pC5JtyZ=42pt|Xvh?Ci
zrFx^9Zt2>Glg;Xo{P1mkgZ0BU=$qEd^Izff?zGUzi*F^%%oIyFgOcucQY2bV5})SB
z(w{vf#`b$8n|~AT4K%nrd^_r@u8uW(FhKZJ2&#U%n|QdMGS$#DSz!+)<(r<3%MZiN
z|Ew-OZa-IgJmTwnr!ij1BCv6PX4hLSm26X7oXpB&&bU(Tx~Fw3LSbCLRpd_fq-DPP
z)!UC)GI&x>$ovLHqYjY^U#l<`8&N!O3v+*M)U@ux<m&!yllO={*=8HAvi;46Z;gQ&
z{HIdbn}cU(x(K%3=<RW~F~m8F)pY(800M`@a#AmGj&;@`GE?IP0=ZQ`G)@*8fv_~0
zc8cdTzx#if3ru{77%D*2qJrp3rx@OgDw<k&yymHZwyK_Z9|iy^cA(mx=OGH)oCsrk
z?Ck+mvkf1N22{0Blfe8*C$EEee`Xkc^vsVSA|fE^!mLeON2~q<c+3hjU--+k?`$Oj
z9oE!5JU03a8%9J1Hj)j0r2NpFvxzOwE$4z}N(_6%efbI-yDBMH{zWl9ojE3cs9;8m
zJGI<8Sy<0eI#M*Nqobh_F(JcLrIn+lsc}A8;%)IdC3U?y<gQx|ST;&Lww~~lKr(e=
z9ME@`t9AMiiQ)N>2%?5Y)LdSr=3XPv#!S_@ao&0<v(&KjKN%|`uH|`@4dR&!Jo3{O
zo6Ni{%U)DDJ``Y9&{)GP>uN+!Fz)j0+aa(U%7|gjs<9r@|HkbQ*Ta9Y;o-s#tZH!j
zgOJfRCKnztdF1|E=eiYdePg7_{#&K}oUNwdnZWTr)&e(jszZYD6+(uRHFx5<rm#h~
z=OpUFXW;aBf2BzlMPXN(iitwx7lV)Ah%w5N4eR#CEy?LVi(23fc2|^z<u$NUDc^NS
z9mnP=@z@J`Hf@$k^CWe)tdE?}bNF5l@fNi0;%4X$SuTx$`laoQKIuhUkr=Lph9vx8
zN%b}T(H0NTIqPOIzx{0SM_58SvC(taW`w7qX`HU=LPlWTXX&tdO@D?&la(iH1HEE3
zg0NrY?m%Rn?fw>{>~aOev9BZi#%)ga_yP(Ii_PT2*-%9!u5n@X;hxt)+Hf28JmDDe
zy#*&n4IR7QPu8C_W0+gc|9*|JIron5DESh5;bUG@KLwCo0+MVB;r~Wb9uKhM=AXXb
zKZF@*Ss>kMOu^Roc@1pW7aY?0_p6{h+a6w*o5-)I?18>3>KQ2IQSju61R>YW3H#U0
zXSGo8Z9Yw1pNmX%1*rw09j>>@Hm3t*>jg0K@(e~rUZ(`-i%HI%PmoI^k3}iYjf0Xn
zU7vmkrEAY04(Zf0Z#-GKw(_lAmtIhlM5DP>sz+Fao)^7Hi=L5wa*~ht$`tYo>9D9R
zrjdI;i9a~SR<VB?wg|1oUgVfHHKVW#xMsTJWt@^30lBYR(*bpS0R5UpJJ9gCkJ<G}
zN7p_T^reL7arS-pn|C6~5w<QXR~kRicl-a4Th^S;z?yZ#`2xIFD6rzt<k=`3R^L|~
zvhW+HD%D@ay72gCY^1xp6_A(OZ6Y3%OSLA|uAi{2fFjB5C)c+Eq_OXVD?CqD!!dv#
zZ>9-T+0956x=!cEc6_@_)(7pAY>K4ic2o>;EYabKc2#gn?`sT%yG38a(s9U_d4sc<
zC5Et8ij|_|+rViH7p3_dq2u*s<o@P1NEMWvKlzW9G=M-B$LECbc2BnK_bL#CE#;9Y
z;sJeAeJD^lKeq(l&^B^kJ~v-muxTVHA<5Pb?6pYP)_PMaa#g;RPNBrDr&8to%H(z2
zIlIQys(quyjCEfs)OcZAY2M_{9eO`a66hU5YP@?j5!h5YX2(3EE&!Zyh1v?_?pO5L
zjgKoc^k+M&%P_LOVUN6B;6LjOf|@X5wb~tJdR1SU_5E^P@TsH#2kn^H1VNiNO*j8A
zCKaw@D4Z>-oO!VC{?;pG-)}mMHtXh*u-Y8An<qWY4wwk%l`nU0MxgoAhjl+*57<91
zcq?rtnKp*R<pksg5Xiqq)Mwy4%oRA0@C?@h&&b;V$hH^XuJkYAv6T;er_q2f>|T4_
zk52~+B9jRW%{uNm98Y=X?K1p$#_uHX%vt5)#^nx!aI%&^;pd>g(}|=bdWf4M{Rx`S
zD!tb|kDdq_QgWwT5dTrupbp=)tt82vL+1Iek&qS8l>0?fq(x$vu_<v(6y87;)S6#|
zw0vUCee;f_Kb_uthSZn5(HUcjny&O`MvBvJ>h7`5yydD9dtWwIOZQPDvr;*%;Igbs
z7&$JPz-a0xsdozd3IH_t77rK)j4jkbfazHRg|Nd5H&&D9wtw8sk#e6NO|zqa_r7XO
z4&$31PZOaY<cBZB1wzYo-#<YF`=~?2MLezv>OQS=f7u&n^X*K!?Q<V*QM~Q(9!DL*
zISRc!-uOv=ZUVL2;WPyU?T0;1%3-%}#hqF$>Wqv%_!+>}O7mX7`PcLBD|y&~i^g3#
zYxqDSsW79T4a*bg0Ugue0Y)IemmWOZ;!}Om2_dZ+X+%JV6z7E*%**nL6xL=B4n@53
z%$6(VMN0x$r0UTmrMR?`kWK$d{LYdnTnahKW3w?Q(@$AE{PjE5*rQ>ZRdeR^JH2`z
ziXzW`h;85{c-{wT-0Msy6}}*}nVEW(36IU9`0@&fq;z{V3m>*^H^43eR9%qf4L<UV
zCC0Owd8(BG9{t63jlw@=Fdt-NQ~px&uP47G3StLEk#c-fFFIauUTmPA#pm>`zdW!F
z!0Q#)yFC+XuDoo~yLZyrBI9q(YO|5ySQMR6yC98geJ7#6wr#2<4=okJ2b6u8+p3h4
zo+maosQ*m_zL)?y3U_(K^}F*aWx}M{KXkOl_!BUY_Z@WBvBKNM?s$dBg+ks_4ge7~
zd$<42p%`!V!#2kcP(hzWIzHXOC`)XjN{+sA9r3vHH?=k|;jSUvRF=X!7|pug#BIk8
z@3MAca7MQ8ZNWN!7V}6nP2Wg~Q|`BL=YUpNSW@ooo<W~=>gAEpY-iBf@MSyNwqtn2
z$B}8=b&z&%OB}>`Wd21N3oqI_HI1<6`e@y^G#VGje*VV54!QhT=fl?@dr~cCtc(>6
z1l&E}-)MQo4NSOK`)y{*+2<x(!#JJJzs!xLo7CZUe+w*ts-Y)rajM9p;qsZzRw=+k
z(vCoZK92lS6lNh}o&r{TC%P9zW)oR@qn$<GY3T<kKibMN2|2xix5aK`LtY?e6m#6%
za}+R>J<DC99K@6Kixseuuos7HJ8RIcaBnSu{47H#xN<l0oh~!sP04ut@IbIOxFc8!
zU)X+4A#9~<>PchUbd1;O+UxDi&0oLkOBmufXU4mT0(bRzW|;WG7lY7yKT7Y-uyZQ+
zA@aVDfBJm9G@He1;LWB3;y}yU=E0BXOoRyiZ>!=xe=2b<rl&;8i^2rYyKEE?q$HBY
z-`p?b)h(Z7?$4QOVeFS)8=iH4FDAWpNPpowS>tDAL!(xLq0?y*ePay6XUs_nRP<-o
z!4kOCFI&`Fs9+AM8IEH%z08kJ)H`se%i%T&M)weA)HRa^-(Srr{PM)wj@v*|ig(vb
zG+GJ+M#5duU9bU$5Mg)>A0o^~fON>`_UC=sRHOh*`c-nO1@Gnu0B`}lPKKMV1ii`)
z#J;?3G0s}O+-nRt^PLSFVlc;b;}q{2h+v&P))(GBqtRI33y<VlVdr#c&kW!BS)x?(
zBtjh4Tgo~cqabUKhXDJa$Mvp$H;=8~;@)TBWM^t-r(psiByUVj>;Qo{3<XS%V5{$k
zMd6%0*fJD$D4l>AHqz9xd&ZZrcB}(wy=YTFhWT{650rXUjYVX~MTVQ7b;^Vf__tD=
z<o3$eed=WS^m+HN@K>N(eL2jeo>Q)e$R)-ZLK;8(p8KpYk69}MNP1b%X)!tEeAN=Q
zsO2{I`r;4YE7*?948xIBK!)W+kv0h*$7(+!A?gsT6Nov<N?=4px31Lzzh}aS!Ap_E
z-sb&8Wk3F{1i9_ZTd-A&OAQEr7vn7Ol21XAydHHKk@YNiA7f-7DKN5hpS{bFM$6ZN
z6e_|~m&rN3S<7?dAU`ALX>MDz^1;`0PVV$>!T##!s9@y=@A}+wLa<|ghN~0>r^}zI
z_3f^d5nFna7PNcc#v3h8*8TcdoVpZmJemey4v_KXPJO>QKkXVj%yCBSFMLPzF6}!~
z?Q~7$uq^?eza}2v{UkyGYhwGeZN5oum~5BFwGk_?=T2i4zw@k_`C?;RKcsipydyVw
zeTwkj7L(j7pWk(B$J{v;a)5R2yO0)PzGknOy}i%ZmgIHg`X?%M{LQqDV!Spi*P^gL
zJzMZ`lZu6EG5gbsaNydf3Z;E{&}B^%XW*H@C`6Utx}SoxKk2$4HNKS_nmD%4TCodc
zxo>sY;$vVf%*`E+vA=G8?X6Nd1lZUmSPR*N8{S0^;P_73W~YW(iGHA747Kw+u!Ykr
z#w;;Er-J(LW+eUN*}^OTYRn&);c6|HUGi<nP@8!?EKMhey|?BI2!Z`J_Chup`>DJ$
zH@BCy0=6~PxO@Xmf^TQmS1gbFo#p<77YSPLxnB^2`Wm#M2j?Vzyd&>NXWb^i*ZQ$U
z@uBX4xL@j4*kXwQlT9M$o^39=%ua;`bU99s16IK4&YK<bRAe8KP967Rr?%LqYS{q5
z0?$N7A%JbkPSk_=4YOCbs(dUv*^c}lg>1DOFPG@f<F^alMo$(%zMy+Nk#IGMTZZ)o
z8qS#{@2yCi?ZVW&%~$*r*+h(?k9&K}C?8ayom))zIW0f+s%~eVd!AZe)HT$d%?oK~
zcEFN8uK#2})1pZAx_P#qp)@f8G1SvHW=4bWAR{Jxeq57qDf&Swa;GT-JO=Hy1Ng6Z
z4a-c%D|gKvnDyeO7b-YS^;<1>I{2P)=ZRT<YH!uXISP2rLz8x{pNQ~YM>Vi{nlPof
z$PvkNWhtD=E249^8(&^7<e1=M@e*OOUU;6F4Oo#{RR2{c|9~L-X|YbQCot6L=h8N}
z7jl(QkoQM}{M6k#Er;n0yRjjDBx0w0TkocZ&;&!b9OqL3xn1}Cclz?}1kIr@_=S0X
zPc0>DZ{XvzI{nOaUKS{KDr{j+JC1qXEEHISfVe>A+<I;hyY$U;_Ojc1vonjX&5@rJ
zCt10RZz6Js&|bT0=6RI-h2f^EE$U@I#jKf#U@<~!Tn3FD8?0COaHedBu!v{@=)Ax3
zx{Z5tkB4VnyC(#~ZGU2=?RN5sA3TZba;W-s>ZYS==6mg>bgEefe}`TeL8Ib^7cY>N
z`}RP;oN5W5os6kZbv!n|JUr*D<&(?ISwiw;wRl{mvhd7_zfdJC)nj1H7q(2aJLR{v
z#fLcmwQhgAdRnb)oxO3S7b^%kID=*nv0Y)|8xSe{JS5Oy^Xu^nw5|{WT9G2TY222>
zn*ES$^9FjxwOoFGYvZX2KdI8t!q`d(5%@h;e)4fbRShz>miW~-%KN#;Op;uGNvP7>
zYjYTJqh_0y!~FGuLR}n#O=JR)4XgbpUE!=j?Gfs?F*f9`u7VY?^XaETKEv`=^6h*g
zXl?b!i_Cf=hio(RC6Rp&dUAm|z(~>#)%cL`$toLgM*rsIX#3c}`K+69Q)@vx=%y}C
zA~Z#rzM#h6V}RfpjHjyj$eAC$b?s7(bYsq(%q(rHGg%pVeM0VW1pZ=a|F-TF`qrrF
z;C`nZwyiX0<7&CgDn6U-z$aX`?!vgNC~kkZ;-gl&_0C4cZT0YBWuccipp@+>^1+T0
znw)}}3O3#?XOe~nhp#Vlx-owVwaQCcl1^+>9^WPHxCKubOL!_aoXwnza$2{(sH?*x
z(e;9H7q7giqoX_*n}1p-?3mZNJaHFRIkF9`-rxDvtQkN`=7bO3PlgDeMw@>#y?6iC
zit{?y6Vblw?oWQ;seLoo+jG9xMNB)C1bsglHN_<KQ~5CZH!bY^yCb1ZhdZ|}u`E8L
zDT!Bg+oDo?fGCP{A(w>!jPaRxSjO{H=FLB6fVzTKtjg)viQC%o@lXji;cN5bt;Fzy
z-hPn-#qIDjqHTn0ijUclreV9ljjJf$E4}e}R!^XR1i)jI7W8C3!nxIIy`Dyiv%{fR
z$8*Ofqh|X2D^*uK{Zh=-%j7rP>85*?v6p)~yzw1!`<z%J&KdlcgO;}1hnzb{Cw>aB
z)c5p;)`t3rpWID=Vs(It>E~c-#5w=w9FqCcxz(H=7fstTHTb4$k%+LC)DqTNn}|@`
zNqgj}!;bfMc67}<$|hb}9fuXye30->BH`5iAF$wTNMuH>Ypjr2(|Bvq(IDgZ4^75<
z+&8-<spWcSEFmT^EmO=E<etm0stLk*6X8IGr0i^ytQ88-m3*D_p+}Ha-xs~qDq~(?
z@2$&1s0o{wN8#Lr>p>jq<f3oQtwVE+4QvwUwe<Q;v4F@&QGtzOE!OJ37osA1%lcDB
z0MqjV?dt*!dGs;}+o8d9Rmyc@#q}?H1IX?R&uNUin&8Z9CT!kY*vlxImOv4&9X5N}
zi=|U-<J9U7n&fR5oxNV-=4?;y%>v0`%UR+M_e$6Bobxgdgr?Ew2gY$8!1g+hv*>ey
z?ve}bAZ{12?T?~?>IsX0!xqpT$&$5^wV?4;m11Kl&*l^%Y!YNABNjm)h)ibuON_{4
zQKg~<2YHRiT90<VyMpEPcRmU&7Xsi}l}sqquJOY1X=~)n&}y?pWL4s2+0#&yr*1wA
z-&=?9nv&YEa_fdx8z;-osB8~J&@x^FBz*#%iHA3TG0eh#q~@+*-Jzcw*h7-M!qT6n
z9Zap>kTjZT3f>8Tj@Ft=jB5C=zQnsA^_~cUo+l1h6djEE2zu|gWbDYq8beBrIzJ_x
zc3zt^ttj56icCe3N|%EVZrq-T{80=Q;j;hS)AZqG9*4xy>%>qWj*b-GvrtbKkdiT)
zNWZj&EWK~*m+j@3EB(l+UA2rZKD0|=+fYvF=1aS~RiVk_fChdC^q+j_4*O>A{Rux7
z&_X?yD1MBeC37Ow_wW<{a4`4dm2TN?w_jy#m)>$`nd=wT&mfqxMl5j7FR){OkJelB
zx+gd~epkXTgMv4?hnQcN45EUWV>LHQy;25}KKhUa?nf)5XdGB3Wv1Dbta<Cj$di|I
z5IV!J8M3LpVasp*^XX!DOx}m&gPOIjZEKz#i@6(0K546`Z<7Tr)f@S(x6F8$$*@H9
zq@@Qy<DRo86gAdcWL6Tz6yQp5JNIruxlx2;aGd?3G5v?!I`eN%gcP2u8+<}nDu@tb
z_Z4;oqeUyr+A(kJmNALMsc|W<ei)}$LAG>fVFfQAD^{OZKqNI^=+3f9BzUwX5ls>2
zm{}ZOF|Ujq5|pgIn_o^ru!u|;u#n-r38+jKt)BqGVbj|-lf0hnFGVAO-p_W`RR=1P
zMROBYDcRvmzSEan9c4*s;ffn^Ngzi@K@N!84}9T;FZ5ngGE1MBE)X0c)3fEA(nU5#
zUak2?jVKv&i94{>i}wJ<jA1GV6)fo)d*So6hPP>;#G4EeJ$k~Tem6&Oam*qHV%uh1
zB~X^{bS`?Vw&i<Ap<U;t8>@oAMq1#TIGAX?VhjNc&$H0WL>dyqm?XA%JUdz2;}knr
zMw};Z1*F8jCbuEUwT}I{`!;KaJf!{pon>T#Mkq=q3Ias=@_kv_fpk*1TU_U+nPTko
zV8tMckFFVY5{J9vX@Seji~!*HF_DzvpDru?&Vo)vDq-eT_1DQ%(j9j1Sf#Cen>Ig|
zSr5%lZ_a(F!$=-cgk10`kdl&h0)B?{R?()UCYy+CNHW&9qzykGz(2@thiA8azs*_4
zt-~EgdsQ5Z?WS8Sww_|<#~0em-7+yFvQVXwCFqLFx!2M8Q)xT)c8GzWC>r{1k#E1^
z*XXm-hse*B>940BuLMxt{Q~8ljk)_4V}@FhS`CRc;zq25b}lr_nxK&l8(w1-2w9|b
zxEh$ud|WkridBi8w;`j;)W#mL!lVaqK}*$WJU(jup*dpe1K=w3aXGqb^|Wa!NbnqW
zyVg%+{*rtUzn4*#Gi6^~TRl$7-wQjQU3B)q0E*llgYrT43du|~@B{U<C>xue-}+Tl
z&EIoFm97eJqXs&csR62hUx!+YWje1kR67m40^RRYQa^5;+4KCdC|D`FC~*EkZ{*1N
zlHI^^eL!$Xam|Sa9YU?dw@>e4>M?s_(;v-WrZ9A5VH*b}uSxry$m{TZd&Mp1<Y5n|
zD9s_WXXuTQh}G{eEdjaxmWj^%b$S8>GwL(jhPofF@xs)ntftj`-CX-4o?d}(zN9Aj
z&V!QOZ4MXYb)kl5@%tM+vExK4yc)jsC+XU09Ts`u>^*?@;gILqboE^)a}Io~76nmz
zLqn@LEXasgDqWJ!Pl0yJph>b~Y-;Q~g&Zu#(gT`OvHuB>7$6O)-I%SW`<XZn-UR>B
zk7=_~P%P+#kni@6Z7>;SBNNU$T*Vf(vSU>p&scPC0MH}(c5e;^ij0GFgQNC70eFV9
zv6L%6_N{xE@n5ah<t8~rOf-EUoiP`#?D(w*2}o_a?s8Fg-MXLMt<~{d9X*5X7gZFU
zwVze+;3;0{DrlHxH4r(FIO#jy6W0yLUlN;i+&LY-N~IAnPCaYA?zH`Oc6`a3ssbYX
zRn*JV;&^9KqEpZ3ntVYjXfdf$2`$_!<eTSV;oZDTt>n#_jLfQXy$w?~+UXkJHUPuH
z0@d>UWnu#r?nyCuA<gwhfBZ6fN{jB<Lk7-@L;z!70Bbidn_$z}YA}5P6WFg1TbXh<
zeywZ%>KKah4uBRMyfA$)IYom=Ci#JDzGBTrXtp^Z!<rQW&xdC*WZ{6n!AAIo+DvSY
zHJ*PEUGwG?x^L5{q7tY#*7JGQdC0xKdOHRWE%o3W97f=B?u)SHTSeX6p{$=arqX)#
z8y9C|59x$S(e<7%?d}nwQZDe2#7XC(Js=3ueu(Pq#w*qmz@OPU*WmpuBS19{{(|kV
zNVPaIH=U<8txgjrn%0(5!>Tt@zemO?=+n4oC@lOv<`auJn^dNx(kLp9Wx|vD2DvXw
z3eHk!fs^<%RhXbcP-+;E$)}nMHa*wZmGk|kB*h}-&>IeXXo}tv1EE~JKG6pqIR^?J
z4|Q4no>p2ohIO7PjI@?W;_qoA8m<<R&G=ap`Ug=dFKN9aS9)brJBZEJu;ppl7%x%C
z%2iIF;MvIQYNLCqWD?$6(}}B6ErB}gu`ZT3_+<Yh9rf2`q?<asRN-o_4uPrsJp*n}
z+o>NXx%MCBnwh`s=k%&yGOc*Rj84Pr(aoXceOCK`VdIfjXI&LhBt#@?jXU4q;fA-L
zdNC!p)sa*``85q}f4?ik-%?JjLqt{&mh}<wbZMcu-rI3ZfL)H@kRvN(Ta>I7yQUK!
zQdDonc8LTZKYuq+A;u-U!GKtwn0nbA!YSWw6vT4{Ru{<CLaP1L+_@!lH7byeIc&q{
z$SOM8Bm5;Zbjgy_rAS_2==_h{$`pwmtBX9Xg&t(+EcX+PjaQqklw18)eDsb_D362=
zrW+e7o-Dao$%&<ZbsPE_(jqcTKV&+|J;O`WZ(H!oMU3nnWe<v5o}dWVDa0O&={-kV
zUHj|?m`U2t1s}yCv2kXAoMy>3lHHJ&aHQ}iQ~A|q%^;*62Nr>ylP9+*87|+h7i#g_
zUjMYO=<uy(p5#D<#-#c$A(R_C-My`27(Q0tj)6C~aJ=K1U!zcIC)q4C|K7IFJ<H9;
zlPet{nFc~}%O1U0zw&u1T<v<MEpR)<zniViJfyND3l7(nwo5ViO2E;;(U7}Ex^V8w
z`#L5}qn2Nr&-C^csZ%ZCq0<)r$L3=>eSM@Nq!uf?lbi0>ct@4-C%L$@y|MQn&L$OD
zl|0Gdr%U0PTUG@AHH0sg#r=%x4eQ;1Q*4zmtX?)?&3h7-$w3Ea<mNArzH4gaic=4w
zA|2>Xy`IIl`*LyDWG7;CaHMEis>h4t><11YHGW+WN8;Sj$$CH4YG|T^p4SJb9D3Hx
zhn)Z6ySOpoynAbX->6l?_ZJ{ase`fs$EVe(GFn@VmhhApZ}!5M0`t*74YaD@a3Iw<
z_|X!Y(X8aYE*@koo(*|h25t(}U%B9O`P;f`){pdlaDL&4Q!ymy0bcX|0XM_Qs%A|z
zCEm3K_;2$Ar1CVzB{r2UqF1QyPm4|O(9T};!ODuiZmvst)ZoYOIc^r=$68Qu|AI;p
zj;s7dPdyxJQ5$~8ui8Agy&AVOH`K1@hvRif919JHe))Uv$vM()rNEIypK%7u9X=Z<
zJ4-(zgti|X$SwERXOBA`j8brxI#_?z()<V5xvF1xN8FA02z$e)frFYp5~|NV0s}WZ
z;78U5z8I$dk6Rg^ao{ke67<}?e@CT4aQ{-+^t|*DJWJW}08g8Irck*j=@C>)QZ~iw
zV`Z`_i}wJRl6-(mebUMznZnOiBpKFr74rI0TLOLndp&>vH;Q+$BFTm-_gv18fElrp
zSwi<eJM%l{+nfp`5LDqvFHv#a=H|4*i+%l_i3k-26;wBgy+YSp+_7;tmnDgfSeiSC
zn-yyzU~a+YM7R#<Wc;X4A{0wqKB7WZwCPvQ`Mf+s6<JRC3fQWpe>>bLwpt3>0e&V5
zX6rTmDL0yl%@*@xC2<g^^H?dg#a)4D_j27o`-s8de`;WTIwnPU<igwqROag3m;l4*
z_t{b|cfS`tcd6*NHhO5PD<L}4T9BU2L8kZwJ>G|F{5Tg-PsRNghDs(1$4Na+%k8bi
zJ+R00QEn)x`28272IhgE#JTb~R!s0AKm|2_jGRw*ax!ZCC7tLeRT$>&U7P4!^<(XX
zUvH#kZnreAiTbMt{BTO4!6xC05K}99-9*G}(|%izWxego2a#*TmG|IzB@yprS3=1d
z!gsE^N&@RYE81!+{1tmB9DBK|Tu-K7vtPvVnyV^M&#G#x1l6}@j(PG}R@`tSxXsCh
z5m?o$5%r*dT5Pzwk!oMDaGQ<Z)v<^Y<^C&6@kg-~Q$sX(5&sBe|4Iy>4r(iaT*612
zLzRTvoIh)b{G(CHV8HqM?MOC;{%CVZ)NsoXC6vg1G^)(!{(O1C4K=@qxKG9Z+VFp%
z*>K7Jzq_X6{UWcHH$B=@M_MaOzuwccFno)wx2>ywW)|~T32@T5DSn{*;No)1TpLHz
z8^sM7OHdHRnX{Y@!a;F4r6|`+S@)lEBizI%$t2t&oqsxpE+(Tk1}*2`@|}hKSZeHW
zR~|p6;(Fn1#a987&w0LbjAI`Eks;W*YrO>Z>Ty<5BKAo74MZeRuqO;0(J`z*KX-ym
zYu^yVdL1$WM8uFVi>nt5Yfc+3znI&O(bM5Ifh!{Z!bCzA7oZKxf3-qzRCKf-$fLz-
z>ElVzA(r^u1lTUDa|m_7Z=M{ZO#k(0mVM{t8-|>3^x`^D&5bc-NC5wse1Gt|M7*N&
zDJy1><3FG_w*M;4Vv=4x4H<ooH6yqb9G5iS@0ar249+#(3yct=b4xA=WUPmuZ>gJ`
zBmv7TzCd~WJwFbyUZlbx#a|V}p@1(D!KR}(el7fbsZY&r@`l_1V>W3?#F&c5FQrG?
z_&A!-ITHrr9l>#jv=mV#X3HHN%z1qOUfys*On<4Z{JH-8-Eo~oDhRs~NB4R1bFybZ
z+1?DIi~8~Rujs+^|0x2K@jz7IpFr3!=F3e&YebSw&&XPxY3kGuUP0VQ8FJWZQP}73
zG2@CFh#Tou7w)>n?)_~X_Se1np=8n`DE9GWdr8SUqDRcPUfwFR>QAGt;lB;eP}9On
zhP_~r{*@}ZLL_aLQlSi3g4bepDAQl^g++j@{y|sZehNp%ra8f@Jr)VfJzx~V8f^H~
z8&@_RlvO79gxKE@bq=v0sPQ?AzZ0>W?}pZ3PP&)jHD3%Q7CnwBEmDll&&h)yuUF*=
zz;LXx+%cB9QS^_u$Jz0J@(D&mU{hrF+BGw$OeyNh_hZFscV4x6^*!~8bDbD>70dJ=
zj&fcGPHYrunp}rB0gX@QUPah!iISH%^Js*Sm?S;#4=nA6r74O1tM8ce6Rjsp*pN4u
zzE-4vi`#py8;Hr9FiRE6ZY|VAz-<)9Q7)u8AdPe_y5w0glD?e|_EP8x>u$!vC|6%k
zZzfP~%+o_`Aoy>2TUsJ6O2)!01|!2aoJCAoF;p7%=e-8slN?Mwr3ISAC0?sJS!22c
ze!>|#Ui2rCRz6ODHkdZZ4?rgRVsSeV@-ZniFl-0}FSmMq=M=R0!ToqK=xj|whd$U!
zOCnZmGbXux9dx+YT`$blP50$M(ef+CcKR*^OA+Y7js-P{yVQYM{whl}(~%w&Mglgv
z9m;=ulr|W_pM5$Lgo2hl^wSc_cakEg@!8rh$PD4S7$YdHY)7Y`BE&+nt%I*rA0Y-t
zI;WydA>COdPADDuM<RWMA*d9XJv78`0iqfZ@gIKP>5GXWVG``z|0&EgQ7;{=4%W_3
zbrwW16XHy_P2|N-Os^6p=?%>Uw0gY(VT|%`0f?~Up0b1tyPO$Q$8a#`@?hhc--$E0
zr#?$270T(CH_>@04SuC`?ThiR<PiK8&hUBqO-uGsXadBNHby+B^2L^v68Hre`vug0
zAOK+OCdH-vWrc1C1dxHO@(~Ll6X!<`q&<-}z9e}V4}d=#)Gn16BF+eo{vSnb9~oxW
zBU(s*;Q?5Fq!%X%M8@uGB<9^wK#1elB{s3Akk7u0cU`=UQH+bmntNg9rm5~&jQfI+
zCImB4d^}Zj3Nu(SY4<DhN>5;`mn}2JKM%7(i#Q<GX=7*06)iFe=2H?6727mjm&i-Q
zXB|y1d#qq9d#GTOOSesh<G?Y-VM!9g%M)TSQN|kOb(1S8h7ktl^!%bMRS2S7`WsKw
zn9~tsGYJzq!^Zqc`Jn?R(#W8YbfZAUz~L!!M6+A;bOPmGT!!6Os6g42NOQ_<FvCo`
zZNL%8H$Sk_2-8_SM}g!=^?yo+06(7irjbNPi?%W)8KIGpMSviKW?>k07we=XW&Ue0
zi3L=3^^oH~qLay$?au*e>?ar-VS`<1u40?{lN0dLwdhck`43#*BTu77iqhy*ZkZvZ
zIw>#Ad=qV$uY2wQ>3Hx;Zsg-qyY(ih+2@p4q}G4s!W$nD)6K_4a4$>KvXlX7jLLUk
z)sE6{(k*nK#fTi?XA4-^(hy&iAvN&f?mu~gNc-@=01hAO)-s-6YAy*P*V5+7uP`Lg
zXaKtP8G^gml@h4R_u=OtmZ)Id6NHTlWuM^C-HvZzpS)0})f<7`ATluRC@K(Mx5gS=
z+7D(3{?hHj{S<pWgWda!P~LqGz6z0YSFHE)i8oP<Dy9LVnI304viLt|kSy7J+uUUX
zfnFpXAg5wxp!8Cljij@^Yg=)sN6~$@h62QixolHDH;Ej1zGj>$A6VKLF{uBGB>ff}
z77m{U`Keu~%792__?(8NePIjaq2;A9cHxSuWC`+NBVNBW83JXiT#b_Md~$y=z|V5~
zB<9i*`8Z3$Ksad}UM+8cZ1CIwqrea1H~-9RT6P1UAtx+e7!yFIi<t$Y4}D{aSx%LX
zK_~_jOAUmC%)ej>;7Fmd?SzR>-8H5CtA|pgUhJ@<Cv)U~dzgiy)zLnbZYI9@X5&@T
z;Ngql!oZ`@<h8f~z^0&qP>C@BrLqC7*CReQ8o^9Y-b62d1a-SRxKy6Pve*ZyV32%#
z$rMj9;EZE_tj*`8SetT+mY9x6Ur`9p53C;NAn#WQ3Zc=@faqu=rD^%GJp9I_BbL(c
zT=0t9O0pS|*oi5!e!FcmvNuh`@3!CzjK>0||HIG0mx%XRO5s8-cxMT^5C4Ya{X0d;
JLh%m){~rwL(-{B&

literal 0
HcmV?d00001

diff --git a/docs/learn/documentation/versioned/hdfs/consumer.md b/docs/learn/documentation/versioned/hadoop/consumer.md
similarity index 99%
rename from docs/learn/documentation/versioned/hdfs/consumer.md
rename to docs/learn/documentation/versioned/hadoop/consumer.md
index 41a1a51544..653dcf9dd7 100644
--- a/docs/learn/documentation/versioned/hdfs/consumer.md
+++ b/docs/learn/documentation/versioned/hadoop/consumer.md
@@ -107,4 +107,4 @@ For the list of all configs, check out the configuration table page [here](../jo
 ### More Information
 [HdfsSystemConsumer design doc](https://issues.apache.org/jira/secure/attachment/12827670/HDFSSystemConsumer.pdf)
 
-## [Security &raquo;](../operations/security.html)
\ No newline at end of file
+## [Writing to HDFS &raquo;](./producer.html)
\ No newline at end of file
diff --git a/docs/learn/documentation/versioned/hadoop/overview.md b/docs/learn/documentation/versioned/hadoop/overview.md
new file mode 100644
index 0000000000..0820127aa7
--- /dev/null
+++ b/docs/learn/documentation/versioned/hadoop/overview.md
@@ -0,0 +1,46 @@
+---
+layout: page
+title: Batch Processing Overview
+---
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+Samza provides a unified data processing model for both stream and batch processing. The primary difference between batch and streaming is whether the input size is bounded or unbounded. Batch data sources are typically bounded (e.g. static files on HDFS), whereas streams are unbounded (e.g. a topic in Kafka). Under the hood, the same highly-efficient stream-processing engine handles both types.
+
+<img src="/img/{{site.version}}/learn/documentation/hadoop/unified_batch_streaming.png" alt="Unified Batch and Streaming" style="max-width: 100%; height: auto;" onclick="window.open(this.src)">
+
+### Unified API for Batch and Streaming
+
+Samza provides a single set of APIs for both batch and stream processing. This unified programming API makes it convenient for you to focus on the processing logic, without treating bounded and unbounded sources differently. Switching between batch and streaming only requires config change, e.g. [Kafka](../api/overview.html) to [HDFS](./consumer.html), instead of any code change.
+
+### Multi-stage Batch Pipeline
+
+Complex data pipelines usually consist multiple stages, with data shuffled (repartitioned) between stages to enable key-based operations such as windowing, aggregation, and join. Samza [high-level API](/startup/preview/index.html) provides an operator named `partitionBy` to create such multi-stage pipelines. Internally, Samza creates a physical stream, called an “intermediate stream”, based on the system configured as in `job.default.system`. Samza repartitions the output of the previous stage by sending it to the intermediate stream with the appropriate partition count and partition key. It then feeds it to the next stage of the pipeline. The lifecycle of intermediate streams is completely managed by Samza so from the user perspective the data shuffling is automatic.
+
+For a single-stage pipeline, dealing with bounded data sets is straightforward: the system consumer “knows” the end of a particular partition, and it will emit end-of-stream token once a partition is complete. Samza will shut down the container when all its input partitions are complete.
+
+For a multi-stage pipeline, however, things become tricky since intermediate streams are often physically unbounded data streams, e.g. Kafka, and the downstream stages don't know when to shut down since unbounded streams don't have an end. To solve this problem, Samza uses in-band end-of-stream control messages in the intermediate stream along with user data messages. The upstream stage broadcasts end-of-stream control messages to every partition of the intermediate stream, and the downstream stage will aggregate the end-of-stream messages for each partition. When one end-of-stream message has been received for every upstream task in a partition, the downstream stage will conclude that the partition has no more messages, and the task will shut down. For pipelines with more than 2 stages, the end-of-stream control messages will be propagated from the source to the last stage, and each stage will perform the end-of-stream aggregation and then shuts down. The following diagram shows the flow:
+
+<img src="/img/{{site.version}}/learn/documentation/hadoop/multi_stage_batch.png" alt="Multi-stage Batch Processing" style="max-width: 100%; height: auto;" onclick="window.open(this.src)">
+
+### State and Fault-tolerance
+
+Samza’s [state management](../container/state-management.html) and [fault-tolerance](../container/checkpointing.html) apply the same to batch. You can use in-memory or RocksDb as your local state store which can be persisted by changelog streams. In case of any container failures, Samza will restart the container by reseeding the local store from changelog streams, and resume processing from the previous checkpoints.
+
+During a job restart, batch processing behaves completely different from streaming. In batch, it is expected to be a re-run and all the internal streams, including intermediate, checkpoint and changelog streams, need to be fresh. Since some systems only support retention-based stream cleanup, e.g. Kafka without deletion enabled, Samza creates a new set of internal streams for each job run. To achieve this, Samza internally generates a unique **run.id** to each job run. The **run.id** is appended to the physical names of the internal streams, which will be used in the job in each run. Samza also performs due diligence to delete/purge the streams from previous run. The cleanup happens when the job is restarted.
+
+## [Reading from HDFS &raquo;](./consumer.html)
\ No newline at end of file
diff --git a/docs/learn/documentation/versioned/hdfs/producer.md b/docs/learn/documentation/versioned/hadoop/producer.md
similarity index 98%
rename from docs/learn/documentation/versioned/hdfs/producer.md
rename to docs/learn/documentation/versioned/hadoop/producer.md
index a157cd8106..a94d5de25f 100644
--- a/docs/learn/documentation/versioned/hdfs/producer.md
+++ b/docs/learn/documentation/versioned/hadoop/producer.md
@@ -66,5 +66,3 @@ systems.hdfs-clickstream.producer.hdfs.write.batch.size.bytes=134217728
 ```
 
 The above configuration assumes a Metrics and Serde implemnetation has been properly configured against the `some-serde-impl` and `some-metrics-impl` labels somewhere else in the same `job.properties` file. Each of these properties has a reasonable default, so you can leave out the ones you don't need to customize for your job run.
-
-## [Reading from HDFS &raquo;](../hdfs/consumer.html)
\ No newline at end of file
diff --git a/docs/learn/documentation/versioned/index.html b/docs/learn/documentation/versioned/index.html
index a710383ebb..e23b29f9d5 100644
--- a/docs/learn/documentation/versioned/index.html
+++ b/docs/learn/documentation/versioned/index.html
@@ -83,12 +83,15 @@ <h4>YARN</h4>
   <li><a href="yarn/yarn-host-affinity.html">Host Affinity & Yarn</a></li>
   <li><a href="yarn/yarn-resource-localization.html">Resource Localization</a></li>
   <li><a href="yarn/yarn-security.html">Yarn Security</a></li>
-  <li><a href="hdfs/producer.html">Writing to HDFS</a></li>
-  <li><a href="hdfs/consumer.html">Reading from HDFS</a></li>
-<!-- TODO write yarn pages
-  <li><a href="">Fault Tolerance</a></li>
-  <li><a href="">Security</a></li>
--->
+
+</ul>
+
+<h4>Hadoop</h4>
+
+<ul class="documentation-list">
+  <li><a href="hadoop/overview.html">Batch Processing Overview</a></li>
+  <li><a href="hadoop/consumer.html">Reading from HDFS</a></li>
+  <li><a href="hadoop/producer.html">Writing to HDFS</a></li>
 </ul>
 
 <h4>Operations</h4>
diff --git a/docs/learn/documentation/versioned/jobs/configuration-table.html b/docs/learn/documentation/versioned/jobs/configuration-table.html
index b6ae0d9e45..ef81887b5d 100644
--- a/docs/learn/documentation/versioned/jobs/configuration-table.html
+++ b/docs/learn/documentation/versioned/jobs/configuration-table.html
@@ -2170,7 +2170,7 @@ <h1>Samza Configuration Reference</h1>
                 </tr>
 
                 <tr>
-                    <th colspan="3" class="section" id="hdfs-system-producer"><a href="../hdfs/producer.html">Writing to HDFS</a></th>
+                    <th colspan="3" class="section" id="hdfs-system-producer"><a href="../hadoop/producer.html">Writing to HDFS</a></th>
                 </tr>
 
                 <tr>
@@ -2210,7 +2210,7 @@ <h1>Samza Configuration Reference</h1>
                 </tr>
 
                 <tr>
-                    <th colspan="3" class="section" id="hdfs-system-consumer"><a href="../hdfs/consumer.html">Reading from HDFS</a></th>
+                    <th colspan="3" class="section" id="hdfs-system-consumer"><a href="../hadoop/consumer.html">Reading from HDFS</a></th>
                 </tr>
 
                 <tr>
diff --git a/docs/learn/documentation/versioned/yarn/yarn-security.md b/docs/learn/documentation/versioned/yarn/yarn-security.md
index 7b66ed8cf8..0aecefe372 100644
--- a/docs/learn/documentation/versioned/yarn/yarn-security.md
+++ b/docs/learn/documentation/versioned/yarn/yarn-security.md
@@ -91,4 +91,3 @@ yarn.token.renewal.interval.seconds=86400
         </property>
     {% endhighlight %}
 
-## [Writing to HDFS &raquo;](../hdfs/producer.html)

From 7887b6d868ee048767c11c2db3a5d38093d9abf9 Mon Sep 17 00:00:00 2001
From: Aditya Toomula <atoomula@atoomula-ld1.linkedin.biz>
Date: Tue, 12 Dec 2017 21:41:37 -0800
Subject: [PATCH 12/36] SAMZA-1502; Make AllSspToSingleTaskGrouper work with
 Yarn and ZK JobCoordinator

Sending a fresh review as I lost the earlier diffs. This is the new approach that we discussed by adding the processor list in the config and passing it to grouper.

Author: Aditya Toomula <atoomula@atoomula-ld1.linkedin.biz>

Reviewers: Yi Pan <nickpan47@gmail.com>, Shanthoosh V <svenkataraman@linkedin.com>

Closes #383 from atoomula/samza
---
 .../AllSspToSingleTaskGrouperFactory.java     |  50 ++++--
 .../standalone/PassthroughJobCoordinator.java |   6 +-
 .../org/apache/samza/config/JobConfig.scala   |   1 +
 .../samza/coordinator/JobModelManager.scala   |  21 ++-
 .../stream/TestAllSspToSingleTaskGrouper.java | 125 ++++++++++++++
 .../TestZkLocalApplicationRunner.java         | 152 +++++++++++++++---
 6 files changed, 315 insertions(+), 40 deletions(-)
 create mode 100644 samza-core/src/test/java/org/apache/samza/container/grouper/stream/TestAllSspToSingleTaskGrouper.java

diff --git a/samza-core/src/main/java/org/apache/samza/container/grouper/stream/AllSspToSingleTaskGrouperFactory.java b/samza-core/src/main/java/org/apache/samza/container/grouper/stream/AllSspToSingleTaskGrouperFactory.java
index 2d2297713a..d3c5080011 100644
--- a/samza-core/src/main/java/org/apache/samza/container/grouper/stream/AllSspToSingleTaskGrouperFactory.java
+++ b/samza-core/src/main/java/org/apache/samza/container/grouper/stream/AllSspToSingleTaskGrouperFactory.java
@@ -19,35 +19,40 @@
 
 package org.apache.samza.container.grouper.stream;
 
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
+import org.apache.samza.config.ConfigException;
 import org.apache.samza.config.JobConfig;
+import org.apache.samza.config.TaskConfigJava;
 import org.apache.samza.container.TaskName;
 import org.apache.samza.system.SystemStreamPartition;
 
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
 
 /**
- * AllSspToSingleTaskGrouper, as the name suggests, assigns all partitions to be consumed by a single TaskInstance
- * This is useful, in case of using load-balanced consumers like the new Kafka consumer, Samza doesn't control the
- * partitions being consumed by a task. Hence, it is assumed that there is only 1 task that processes all messages,
- * irrespective of which partition it belongs to.
- * This also implies that container and tasks are synonymous when this grouper is used. Taskname(s) has to be globally
- * unique within a given job.
+ * AllSspToSingleTaskGrouper creates TaskInstances equal to the number of containers and assigns all partitions to be
+ * consumed by each TaskInstance. This is useful, in case of using load-balanced consumers like the high-level Kafka
+ * consumer and Kinesis consumer, where Samza doesn't control the partitions being consumed by the task.
  *
- * Note: This grouper does not take in broadcast streams yet.
+ * Note that this grouper does not take in broadcast streams yet.
  */
+
 class AllSspToSingleTaskGrouper implements SystemStreamPartitionGrouper {
-  private final int containerId;
+  private final List<String> processorList;
 
-  public AllSspToSingleTaskGrouper(int containerId) {
-    this.containerId = containerId;
+  public AllSspToSingleTaskGrouper(List<String> processorList) {
+    this.processorList = processorList;
   }
 
   @Override
   public Map<TaskName, Set<SystemStreamPartition>> group(final Set<SystemStreamPartition> ssps) {
+    Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<>();
+
     if (ssps == null) {
       throw new SamzaException("ssp set cannot be null!");
     }
@@ -55,15 +60,28 @@ public Map<TaskName, Set<SystemStreamPartition>> group(final Set<SystemStreamPar
       throw new SamzaException("Cannot process stream task with no input system stream partitions");
     }
 
-    final TaskName taskName = new TaskName(String.format("Task-%s", String.valueOf(containerId)));
+    processorList.forEach(processor -> {
+        // Create a task name for each processor and assign all partitions to each task name.
+        final TaskName taskName = new TaskName(String.format("Task-%s", processor));
+        groupedMap.put(taskName, ssps);
+      });
 
-    return Collections.singletonMap(taskName, ssps);
+    return groupedMap;
   }
 }
 
 public class AllSspToSingleTaskGrouperFactory implements SystemStreamPartitionGrouperFactory {
   @Override
   public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) {
-    return new AllSspToSingleTaskGrouper(config.getInt(JobConfig.PROCESSOR_ID()));
+    if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) {
+      throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams.");
+    }
+
+    String processors = config.get(JobConfig.PROCESSOR_LIST());
+    List<String> processorList = Arrays.asList(processors.split(","));
+    if (processorList.isEmpty()) {
+      throw new SamzaException("processor list cannot be empty!");
+    }
+    return new AllSspToSingleTaskGrouper(processorList);
   }
 }
\ No newline at end of file
diff --git a/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java b/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java
index 87a1cfa3fc..5147169dca 100644
--- a/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java
+++ b/samza-core/src/main/java/org/apache/samza/standalone/PassthroughJobCoordinator.java
@@ -23,6 +23,7 @@
 import org.apache.samza.config.Config;
 import org.apache.samza.config.ConfigException;
 import org.apache.samza.config.JavaSystemConfig;
+import org.apache.samza.config.JobConfig;
 import org.apache.samza.coordinator.JobCoordinator;
 import org.apache.samza.coordinator.JobModelManager;
 import org.apache.samza.job.model.JobModel;
@@ -125,13 +126,16 @@ public JobModel getJobModel() {
     StreamMetadataCache streamMetadataCache = new StreamMetadataCache(
         Util.<String, SystemAdmin>javaMapAsScalaMap(systemAdmins), 5000, SystemClock.instance());
 
+    String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID()));
+
     /** TODO:
      Locality Manager seems to be required in JC for reading locality info and grouping tasks intelligently and also,
      in SamzaContainer for writing locality info to the coordinator stream. This closely couples together
      TaskNameGrouper with the LocalityManager! Hence, groupers should be a property of the jobcoordinator
      (job.coordinator.task.grouper, instead of task.systemstreampartition.grouper)
      */
-    return JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache, null);
+    return JobModelManager.readJobModel(this.config, Collections.emptyMap(), null, streamMetadataCache,
+        Collections.singletonList(containerId));
   }
 
   @Override
diff --git a/samza-core/src/main/scala/org/apache/samza/config/JobConfig.scala b/samza-core/src/main/scala/org/apache/samza/config/JobConfig.scala
index 083dbaf124..de8391936a 100644
--- a/samza-core/src/main/scala/org/apache/samza/config/JobConfig.scala
+++ b/samza-core/src/main/scala/org/apache/samza/config/JobConfig.scala
@@ -79,6 +79,7 @@ object JobConfig {
 
   // Processor Config Constants
   val PROCESSOR_ID = "processor.id"
+  val PROCESSOR_LIST = "processor.list"
 
   implicit def Config2Job(config: Config) = new JobConfig(config)
 
diff --git a/samza-core/src/main/scala/org/apache/samza/coordinator/JobModelManager.scala b/samza-core/src/main/scala/org/apache/samza/coordinator/JobModelManager.scala
index c2e06652d5..99b1abe40e 100644
--- a/samza-core/src/main/scala/org/apache/samza/coordinator/JobModelManager.scala
+++ b/samza-core/src/main/scala/org/apache/samza/coordinator/JobModelManager.scala
@@ -24,7 +24,9 @@ import java.util
 import java.util.concurrent.atomic.AtomicReference
 
 import org.apache.samza.config.ClusterManagerConfig
+import org.apache.samza.config.JobConfig
 import org.apache.samza.config.JobConfig.Config2Job
+import org.apache.samza.config.MapConfig
 import org.apache.samza.config.SystemConfig.Config2System
 import org.apache.samza.config.TaskConfig.Config2Task
 import org.apache.samza.config.Config
@@ -49,6 +51,7 @@ import org.apache.samza.util.Util
 import org.apache.samza.{Partition, PartitionChangeException, SamzaException}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
 
 /**
  * Helper companion object that is responsible for wiring up a JobModelManager
@@ -104,7 +107,15 @@ object JobModelManager extends Logging {
 
     val streamMetadataCache = new StreamMetadataCache(systemAdmins = systemAdmins, cacheTTLms = 0)
     val previousChangelogPartitionMapping = changelogManager.readChangeLogPartitionMapping()
-    val jobModelManager = getJobModelManager(config, previousChangelogPartitionMapping, localityManager, streamMetadataCache, null)
+
+    val processorList = new ListBuffer[String]()
+    val containerCount = new JobConfig(config).getContainerCount
+    for (i <- 0 until containerCount) {
+      processorList += i.toString
+    }
+
+    val jobModelManager = getJobModelManager(config, previousChangelogPartitionMapping, localityManager,
+      streamMetadataCache, processorList.toList.asJava)
     val jobModel = jobModelManager.jobModel
     // Save the changelog mapping back to the ChangelogPartitionmanager
     // newChangelogPartitionMapping is the merging of all current task:changelog
@@ -211,7 +222,13 @@ object JobModelManager extends Logging {
                    containerIds: java.util.List[String]): JobModel = {
     // Do grouping to fetch TaskName to SSP mapping
     val allSystemStreamPartitions = getMatchedInputStreamPartitions(config, streamMetadataCache)
-    val grouper = getSystemStreamPartitionGrouper(config)
+
+    // processor list is required by some of the groupers. So, let's pass them as part of the config.
+    // Copy the config and add the processor list to the config copy.
+    val configMap = new util.HashMap[String, String](config)
+    configMap.put(JobConfig.PROCESSOR_LIST, String.join(",", containerIds))
+    val grouper = getSystemStreamPartitionGrouper(new MapConfig(configMap))
+
     val groups = grouper.group(allSystemStreamPartitions.asJava)
     info("SystemStreamPartitionGrouper %s has grouped the SystemStreamPartitions into %d tasks with the following taskNames: %s" format(grouper, groups.size(), groups.keySet()))
 
diff --git a/samza-core/src/test/java/org/apache/samza/container/grouper/stream/TestAllSspToSingleTaskGrouper.java b/samza-core/src/test/java/org/apache/samza/container/grouper/stream/TestAllSspToSingleTaskGrouper.java
new file mode 100644
index 0000000000..fa3b33f90b
--- /dev/null
+++ b/samza-core/src/test/java/org/apache/samza/container/grouper/stream/TestAllSspToSingleTaskGrouper.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.container.grouper.stream;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.samza.Partition;
+import org.apache.samza.SamzaException;
+import org.apache.samza.config.Config;
+import org.apache.samza.config.ConfigException;
+import org.apache.samza.config.MapConfig;
+import org.apache.samza.container.TaskName;
+import org.apache.samza.system.SystemStreamPartition;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+
+public class TestAllSspToSingleTaskGrouper {
+  private SystemStreamPartition aa0 = new SystemStreamPartition("SystemA", "StreamA", new Partition(0));
+  private SystemStreamPartition aa1 = new SystemStreamPartition("SystemA", "StreamA", new Partition(1));
+  private SystemStreamPartition aa2 = new SystemStreamPartition("SystemA", "StreamA", new Partition(2));
+  private SystemStreamPartition ab0 = new SystemStreamPartition("SystemA", "StreamB", new Partition(0));
+  private AllSspToSingleTaskGrouperFactory grouperFactory = new AllSspToSingleTaskGrouperFactory();
+
+  @Test
+  public void testLocalStreamGroupedCorrectlyForYarn() {
+    HashSet<SystemStreamPartition> allSSPs = new HashSet<>();
+    HashMap<String, String> configMap = new HashMap<>();
+
+    configMap.put("job.container.count", "2");
+    configMap.put("processor.list", "0,1");
+
+    Config config = new MapConfig(configMap);
+
+    SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(config);
+
+    Collections.addAll(allSSPs, aa0, aa1, aa2, ab0);
+    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
+    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<>();
+
+    HashSet<SystemStreamPartition> partitions = new HashSet<>();
+    partitions.add(aa0);
+    partitions.add(aa1);
+    partitions.add(aa2);
+    partitions.add(ab0);
+    expectedResult.put(new TaskName("Task-0"), partitions);
+    expectedResult.put(new TaskName("Task-1"), partitions);
+
+    assertEquals(expectedResult, result);
+  }
+
+  @Test
+  public void testLocalStreamGroupedCorrectlyForPassthru() {
+    HashSet<SystemStreamPartition> allSSPs = new HashSet<>();
+    HashMap<String, String> configMap = new HashMap<>();
+
+    configMap.put("job.coordinator.factory", "org.apache.samza.standalone.PassthroughJobCoordinatorFactory");
+    configMap.put("processor.id", "1");
+    configMap.put("processor.list", configMap.get("processor.id"));
+
+    Config config = new MapConfig(configMap);
+
+    SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(config);
+
+    Collections.addAll(allSSPs, aa0, aa1, aa2, ab0);
+    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
+    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<>();
+
+    HashSet<SystemStreamPartition> partitions = new HashSet<>();
+    partitions.add(aa0);
+    partitions.add(aa1);
+    partitions.add(aa2);
+    partitions.add(ab0);
+    expectedResult.put(new TaskName("Task-1"), partitions);
+
+    assertEquals(expectedResult, result);
+  }
+
+  @Test(expected = SamzaException.class)
+  public void testLocalStreamWithEmptySsps() {
+    HashSet<SystemStreamPartition> allSSPs = new HashSet<>();
+    HashMap<String, String> configMap = new HashMap<>();
+
+    configMap.put("job.coordinator.factory", "org.apache.samza.standalone.PassthroughJobCoordinatorFactory");
+    configMap.put("processor.list", "1");
+    Config config = new MapConfig(configMap);
+
+    SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(config);
+
+    grouper.group(allSSPs);
+  }
+
+  @Test(expected = ConfigException.class)
+  public void testLocalStreamWithBroadcastStream() {
+    HashMap<String, String> configMap = new HashMap<>();
+
+    configMap.put("task.broadcast.inputs", "test.stream#0");
+    configMap.put("processor.list", "1");
+    Config config = new MapConfig(configMap);
+
+    grouperFactory.getSystemStreamPartitionGrouper(config);
+  }
+
+}
\ No newline at end of file
diff --git a/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java b/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java
index eb087bb3d1..9c5dad5f54 100644
--- a/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java
+++ b/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java
@@ -40,8 +40,10 @@
 import org.apache.samza.config.TaskConfig;
 import org.apache.samza.config.TaskConfigJava;
 import org.apache.samza.config.ZkConfig;
+import org.apache.samza.container.TaskName;
 import org.apache.samza.job.ApplicationStatus;
 import org.apache.samza.job.model.JobModel;
+import org.apache.samza.job.model.TaskModel;
 import org.apache.samza.operators.MessageStream;
 import org.apache.samza.operators.OutputStream;
 import org.apache.samza.operators.StreamGraph;
@@ -97,6 +99,8 @@ public class TestZkLocalApplicationRunner extends StandaloneIntegrationTestHarne
 
   private String inputKafkaTopic;
   private String outputKafkaTopic;
+  private String inputSinglePartitionKafkaTopic;
+  private String outputSinglePartitionKafkaTopic;
   private ZkUtils zkUtils;
   private ApplicationConfig applicationConfig1;
   private ApplicationConfig applicationConfig2;
@@ -113,7 +117,7 @@ public class TestZkLocalApplicationRunner extends StandaloneIntegrationTestHarne
   @Rule
   public final ExpectedException expectedException = ExpectedException.none();
 
-//  @Override
+  //  @Override
   public void setUp() {
     super.setUp();
     String uniqueTestId = UUID.randomUUID().toString();
@@ -121,6 +125,8 @@ public void setUp() {
     testStreamAppId = String.format("test-app-id-%s", uniqueTestId);
     inputKafkaTopic = String.format("test-input-topic-%s", uniqueTestId);
     outputKafkaTopic = String.format("test-output-topic-%s", uniqueTestId);
+    inputSinglePartitionKafkaTopic = String.format("test-input-single-partition-topic-%s", uniqueTestId);
+    outputSinglePartitionKafkaTopic = String.format("test-output-single-partition-topic-%s", uniqueTestId);
 
     // Set up stream application config map with the given testStreamAppName, testStreamAppId and test kafka system
     // TODO: processorId should typically come up from a processorID generator as processor.id will be deprecated in 0.14.0+
@@ -147,15 +153,23 @@ public void setUp() {
       LOGGER.info("Creating kafka topic: {}.", kafkaTopic);
       TestUtils.createTopic(zkUtils(), kafkaTopic, 5, 1, servers(), new Properties());
     }
+    for (String kafkaTopic : ImmutableList.of(inputSinglePartitionKafkaTopic, outputSinglePartitionKafkaTopic)) {
+      LOGGER.info("Creating kafka topic: {}.", kafkaTopic);
+      TestUtils.createTopic(zkUtils(), kafkaTopic, 1, 1, servers(), new Properties());
+    }
   }
 
-//  @Override
+  //  @Override
   public void tearDown() {
     if (zookeeper().zookeeper().isRunning()) {
       for (String kafkaTopic : ImmutableList.of(inputKafkaTopic, outputKafkaTopic)) {
         LOGGER.info("Deleting kafka topic: {}.", kafkaTopic);
         AdminUtils.deleteTopic(zkUtils(), kafkaTopic);
       }
+      for (String kafkaTopic : ImmutableList.of(inputSinglePartitionKafkaTopic, outputSinglePartitionKafkaTopic)) {
+        LOGGER.info("Deleting kafka topic: {}.", kafkaTopic);
+        AdminUtils.deleteTopic(zkUtils(), kafkaTopic);
+      }
       zkUtils.close();
       super.tearDown();
     }
@@ -175,7 +189,7 @@ private void publishKafkaEvents(String topic, int numEvents, String streamProces
   }
 
   private Map<String, String> buildStreamApplicationConfigMap(String systemName, String inputTopic,
-                                                              String appName, String appId) {
+      String appName, String appId) {
     Map<String, String> samzaContainerConfig = ImmutableMap.<String, String>builder()
         .put(TaskConfig.INPUT_STREAMS(), inputTopic)
         .put(JobConfig.JOB_DEFAULT_SYSTEM(), systemName)
@@ -197,24 +211,25 @@ private Map<String, String> buildStreamApplicationConfigMap(String systemName, S
     return applicationConfig;
   }
 
+  /**
+   * sspGrouper is set to GroupBySystemStreamPartitionFactory.
+   * Run a stream application(streamApp1) consuming messages from input topic(effectively one container).
+   *
+   * In the callback triggered by streamApp1 after processing a message, bring up an another stream application(streamApp2).
+   *
+   * Assertions:
+   *           A) JobModel generated before and after the addition of streamApp2 should be equal.
+   *           B) Second stream application(streamApp2) should not join the group and process any message.
+   */
+
   //@Test
   public void shouldStopNewProcessorsJoiningGroupWhenNumContainersIsGreaterThanNumTasks() throws InterruptedException {
-    /**
-     * sspGrouper is set to AllSspToSingleTaskGrouperFactory for this test case(All ssp's from input kafka topic are mapped to a single task).
-     * Run a stream application(streamApp1) consuming messages from input topic(effectively one container).
-     *
-     * In the callback triggered by streamApp1 after processing a message, bring up an another stream application(streamApp2).
-     *
-     * Assertions:
-     *           A) JobModel generated before and after the addition of streamApp2 should be equal.
-     *           B) Second stream application(streamApp2) should not join the group and process any message.
-     */
-
     // Set up kafka topics.
-    publishKafkaEvents(inputKafkaTopic, NUM_KAFKA_EVENTS * 2, PROCESSOR_IDS[0]);
+    publishKafkaEvents(inputSinglePartitionKafkaTopic, NUM_KAFKA_EVENTS * 2, PROCESSOR_IDS[0]);
 
     // Configuration, verification variables
-    MapConfig testConfig = new MapConfig(ImmutableMap.of(JobConfig.SSP_GROUPER_FACTORY(), "org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory", JobConfig.JOB_DEBOUNCE_TIME_MS(), "10"));
+    MapConfig testConfig = new MapConfig(ImmutableMap.of(JobConfig.SSP_GROUPER_FACTORY(),
+        "org.apache.samza.container.grouper.stream.GroupBySystemStreamPartitionFactory", JobConfig.JOB_DEBOUNCE_TIME_MS(), "10"));
     // Declared as final array to update it from streamApplication callback(Variable should be declared final to access in lambda block).
     final JobModel[] previousJobModel = new JobModel[1];
     final String[] previousJobModelVersion = new String[1];
@@ -231,7 +246,8 @@ public void shouldStopNewProcessorsJoiningGroupWhenNumContainersIsGreaterThanNum
     // Set up stream app 2.
     CountDownLatch processedMessagesLatch = new CountDownLatch(NUM_KAFKA_EVENTS);
     LocalApplicationRunner localApplicationRunner2 = new LocalApplicationRunner(new MapConfig(applicationConfig2, testConfig));
-    StreamApplication streamApp2 = new TestStreamApplication(inputKafkaTopic, outputKafkaTopic, processedMessagesLatch, null, null);
+    StreamApplication streamApp2 = new TestStreamApplication(inputSinglePartitionKafkaTopic, outputSinglePartitionKafkaTopic,
+        processedMessagesLatch, null, null);
 
     // Callback handler for streamApp1.
     StreamApplicationCallback streamApplicationCallback = message -> {
@@ -251,7 +267,8 @@ public void shouldStopNewProcessorsJoiningGroupWhenNumContainersIsGreaterThanNum
 
     // Set up stream app 1.
     LocalApplicationRunner localApplicationRunner1 = new LocalApplicationRunner(new MapConfig(applicationConfig1, testConfig));
-    StreamApplication streamApp1 = new TestStreamApplication(inputKafkaTopic, outputKafkaTopic, null, streamApplicationCallback, kafkaEventsConsumedLatch);
+    StreamApplication streamApp1 = new TestStreamApplication(inputSinglePartitionKafkaTopic, outputSinglePartitionKafkaTopic,
+        null, streamApplicationCallback, kafkaEventsConsumedLatch);
     localApplicationRunner1.run(streamApp1);
 
     kafkaEventsConsumedLatch.await();
@@ -268,6 +285,99 @@ public void shouldStopNewProcessorsJoiningGroupWhenNumContainersIsGreaterThanNum
     assertEquals(NUM_KAFKA_EVENTS, processedMessagesLatch.getCount());
   }
 
+  /**
+   * sspGrouper is set to AllSspToSingleTaskGrouperFactory (All ssps from input kafka topic are mapped to a single task per container).
+   * AllSspToSingleTaskGrouperFactory should be used only with high-level consumers which do the partition management
+   * by themselves. Using the factory with the consumers that do not do the partition management will result in
+   * each processor/task consuming all the messages from all the partitions.
+   * Run a stream application(streamApp1) consuming messages from input topic(effectively one container).
+   *
+   * In the callback triggered by streamApp1 after processing a message, bring up an another stream application(streamApp2).
+   *
+   * Assertions:
+   *           A) JobModel generated before and after the addition of streamApp2 should not be equal.
+   *           B) Second stream application(streamApp2) should join the group and process all the messages.
+   */
+
+  //@Test
+  public void shouldUpdateJobModelWhenNewProcessorJoiningGroupUsingAllSspToSingleTaskGrouperFactory() throws InterruptedException {
+    // Set up kafka topics.
+    publishKafkaEvents(inputKafkaTopic, NUM_KAFKA_EVENTS * 2, PROCESSOR_IDS[0]);
+
+    // Configuration, verification variables
+    MapConfig testConfig = new MapConfig(ImmutableMap.of(JobConfig.SSP_GROUPER_FACTORY(), "org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory", JobConfig.JOB_DEBOUNCE_TIME_MS(), "10"));
+    // Declared as final array to update it from streamApplication callback(Variable should be declared final to access in lambda block).
+    final JobModel[] previousJobModel = new JobModel[1];
+    final String[] previousJobModelVersion = new String[1];
+    AtomicBoolean hasSecondProcessorJoined = new AtomicBoolean(false);
+    final CountDownLatch secondProcessorRegistered = new CountDownLatch(1);
+
+    zkUtils.subscribeToProcessorChange((parentPath, currentChilds) -> {
+        // When streamApp2 with id: PROCESSOR_IDS[1] is registered, start processing message in streamApp1.
+        if (currentChilds.contains(PROCESSOR_IDS[1])) {
+          secondProcessorRegistered.countDown();
+        }
+      });
+
+    // Set up streamApp2.
+    CountDownLatch processedMessagesLatch = new CountDownLatch(NUM_KAFKA_EVENTS * 2);
+    LocalApplicationRunner localApplicationRunner2 = new LocalApplicationRunner(new MapConfig(applicationConfig2, testConfig));
+    StreamApplication streamApp2 = new TestStreamApplication(inputKafkaTopic, outputKafkaTopic, processedMessagesLatch, null, null);
+
+    // Callback handler for streamApp1.
+    StreamApplicationCallback streamApplicationCallback = message -> {
+      if (hasSecondProcessorJoined.compareAndSet(false, true)) {
+        previousJobModelVersion[0] = zkUtils.getJobModelVersion();
+        previousJobModel[0] = zkUtils.getJobModel(previousJobModelVersion[0]);
+        localApplicationRunner2.run(streamApp2);
+        try {
+          // Wait for streamApp2 to register with zookeeper.
+          secondProcessorRegistered.await();
+        } catch (InterruptedException e) {
+        }
+      }
+    };
+
+    // This is the latch for the messages received by streamApp1. Since streamApp1 is run first, it gets one event
+    // redelivered due to re-balancing done by Zk after the streamApp2 joins (See the callback above).
+    CountDownLatch kafkaEventsConsumedLatch = new CountDownLatch(NUM_KAFKA_EVENTS * 2 + 1);
+
+    // Set up stream app 1.
+    LocalApplicationRunner localApplicationRunner1 = new LocalApplicationRunner(new MapConfig(applicationConfig1, testConfig));
+    StreamApplication streamApp1 = new TestStreamApplication(inputKafkaTopic, outputKafkaTopic, null,
+        streamApplicationCallback, kafkaEventsConsumedLatch);
+    localApplicationRunner1.run(streamApp1);
+
+    kafkaEventsConsumedLatch.await();
+
+    String currentJobModelVersion = zkUtils.getJobModelVersion();
+    JobModel updatedJobModel = zkUtils.getJobModel(currentJobModelVersion);
+
+    // JobModelVersion check to verify that leader publishes new jobModel.
+    assertTrue(Integer.parseInt(previousJobModelVersion[0]) < Integer.parseInt(currentJobModelVersion));
+
+    // Job model before and after the addition of second stream processor should not be the same.
+    assertTrue(!previousJobModel[0].equals(updatedJobModel));
+
+    // Task names in the job model should be different but the set of partitions should be the same and each task name
+    // should be assigned to a different container.
+    assertEquals(previousJobModel[0].getContainers().get(PROCESSOR_IDS[0]).getTasks().size(), 1);
+    assertEquals(updatedJobModel.getContainers().get(PROCESSOR_IDS[0]).getTasks().size(), 1);
+    assertEquals(updatedJobModel.getContainers().get(PROCESSOR_IDS[1]).getTasks().size(), 1);
+    Map<TaskName, TaskModel> updatedTaskModelMap1 = updatedJobModel.getContainers().get(PROCESSOR_IDS[0]).getTasks();
+    Map<TaskName, TaskModel> updatedTaskModelMap2 = updatedJobModel.getContainers().get(PROCESSOR_IDS[1]).getTasks();
+    assertEquals(updatedTaskModelMap1.size(), 1);
+    assertEquals(updatedTaskModelMap2.size(), 1);
+
+    TaskModel taskModel1 = updatedTaskModelMap1.values().stream().findFirst().get();
+    TaskModel taskModel2 = updatedTaskModelMap2.values().stream().findFirst().get();
+    assertEquals(taskModel1.getSystemStreamPartitions(), taskModel2.getSystemStreamPartitions());
+    assertTrue(!taskModel1.getTaskName().getTaskName().equals(taskModel2.getTaskName().getTaskName()));
+
+    // TODO: After SAMZA-1364 add assertion for localApplicationRunner2.status(streamApp)
+    processedMessagesLatch.await();
+  }
+
   //@Test
   public void shouldReElectLeaderWhenLeaderDies() throws InterruptedException {
     // Set up kafka topics.
@@ -495,7 +605,7 @@ public String toString() {
     }
 
     static TestKafkaEvent fromString(String message) {
-      String[] messageComponents = message.split("|");
+      String[] messageComponents = message.split("\\|");
       return new TestKafkaEvent(messageComponents[0], messageComponents[1]);
     }
   }
@@ -513,8 +623,8 @@ private static class TestStreamApplication implements StreamApplication {
     private final CountDownLatch kafkaEventsConsumedLatch;
 
     TestStreamApplication(String inputTopic, String outputTopic,
-                          CountDownLatch processedMessagesLatch,
-                          StreamApplicationCallback streamApplicationCallback, CountDownLatch kafkaEventsConsumedLatch) {
+        CountDownLatch processedMessagesLatch,
+        StreamApplicationCallback streamApplicationCallback, CountDownLatch kafkaEventsConsumedLatch) {
       this.inputTopic = inputTopic;
       this.outputTopic = outputTopic;
       this.processedMessagesLatch = processedMessagesLatch;

From 2e5ceec19f52b300b98efb06b292cf9a3db60388 Mon Sep 17 00:00:00 2001
From: Daniel Chen <29577458+dxichen@users.noreply.github.com>
Date: Tue, 12 Dec 2017 21:47:38 -0800
Subject: [PATCH 13/36] SAMZA-1437; Added Eventhub producer and consumer docs

Still need to add tutorials, and configs to configurations table

vjagadish1989  for review

Author: Daniel Chen <29577458+dxichen@users.noreply.github.com>

Reviewers: Jagadish <jagadish@apache.org>

Closes #382 from dxichen/eventhub-docs
---
 .../azure/eventhub_send_methods.png           | Bin 0 -> 52427 bytes
 .../versioned/azure/eventhubs.md              | 197 ++++++++++++++++++
 docs/learn/documentation/versioned/index.html |   6 +
 .../versioned/jobs/configuration-table.html   |  87 ++++++++
 4 files changed, 290 insertions(+)
 create mode 100644 docs/learn/documentation/azure/eventhub_send_methods.png
 create mode 100644 docs/learn/documentation/versioned/azure/eventhubs.md

diff --git a/docs/learn/documentation/azure/eventhub_send_methods.png b/docs/learn/documentation/azure/eventhub_send_methods.png
new file mode 100644
index 0000000000000000000000000000000000000000..477553a23fe3bc3216a73b4ff709e0f16ee14b41
GIT binary patch
literal 52427
zcmd43cT|(<*DkD%jua6D5e0<-DGCZI0s;a`?>!J&Fe;sZh;$Mhr3ffZ0RbUOCkYTr
zKw2__iim)e1V{u#q&F!^AR*y<qBFlU?>Xl^YklR9j}_NS^4xXb``&wB*WNMF+R})R
zN0evJo;`df#`?B<_WXtc{`qnJ4t%3a5;oqm=l&iO{p<GO<fS&g*TNocbV6_XvuG2Z
zbzwh=>Ekh`{P@zVb8#`3c<jzzzWNHvFLO%Wq|3(C{OaAI2Sd;F7Jh3svD?gOFOoTN
zb>$mK|MY~ep5EXWv5Pg<HjNs649g7{Hm%Kz)QxcWi_vzQNN)4;?D+(*&WzGt4C~Ws
zIti@f|9^kip4jgGl^k^*Ja+l+n}zpXRE4}Mlsm2o@dj1evgoL$at=&<U5Ee;%E}P@
zUDy+k1O{3%c6SkVtgt_jR7RlQNmk>{kii!_eRk`+l~<tb1(Abm?)IFGdMgKuLB52%
z`Pefhbk;2)R!~>NIbb8=^6tjXM1K}aBw^hh!fE&6_BgALzcq+DW5Vm8dfq}_#uY52
z+kDBz;>K*XFJu2Mh;xK<({qIvl*$G{2Gcil1O_^4@d(h=Wq8i}n99At<H)@tYWna1
z@b<SmV-It)hn~0!2<f7vnjH}ygZg7tf5mihTKQVrhty<?5DgU(+>#(IfACYh){k$H
zo$OP<!<WCm4gw8!Z6~+#xy}jd`ktWx`-gNt?f{<c0VTz=WmEGezhw#7cBc+FJ$k(l
z_|~4g`A)JI`TNZ*fq`WCs)&eurwMF@Qp^EPm&iYw%M@bXsHPeX2<S!zsm`l%v86Z<
zHjmoCM#$9&;>?LPEjL$;pssQ8tjTaS0(v1vlGC6${kuBqYC*h!uJgo1^YZ)5Cqi3h
zob`X^1CAs7f#QeP7i417ZjQS4ZMbm0Z?<>HD-EuemyCUu&qqZ=PJVRf2fn^<sjqxv
z#(?UnsLd@|%%{v<?VtO>U6AvnH;s4@@&#BPE<V{(m&<3kt#WWXnX{+Qe}2KP$8{_^
zs;b<rG2xOtm6mDY==Mc`I$6yNJieysguT15Wh&?ByjuoqlQk2M%1I8?H5QYv%v-(Y
z%vlZjG#Z`j*)XkZ?jMMdV+?i&>c@=iL~bUC_?@;292JMbE7c?EW99dYGdVpe#kimY
z%3QwzJ?sBufSwGJBahHXlvP7N<mfSY$fR9=b^u$i=lf%%|Kpj;fFwvH^?Yomz`#YP
zv7u`v!a%o|`vP3sBJ88YT{af-0*%zh3F?f_U`L}S(Qc8Lo!gtt(%~Zxx?CL(f66MC
zK~y?awOK&CKp79$%xx?^`w@<&#Zq~B3m%C_Osa6ko)o&cCc?!Tnrm3C^}-)YXfaCE
zWE>VaDwlyPPYC+JqdjN6uaF;PT9|2mE98X&=g~0z^Q4mZ1J4Ak3C(Ndl;*FxU_kVh
z^g2~o9sxBkv;*SE*cgnh^|@&TxsxeK%I_uUXv{Gei%wTZzwVs=p&*_@+gj1!tRjDa
z;SKL7#qWdwo$f=dSV`}XEax0~eZIY!{y3x`ryyG7ewB7NwR`vKdiGr1uvS6D3YUUp
zv7dO@VEFl%SHMg!9hGCK$q(0Sn@ft6>P@*9_wG_Db3t-&t;ELMiA32k_R)4zuKB&d
z0-sDNafG?)3^cA%R;(2OL^l7N>9ja}nY1>{F|3hDOx2gG_`1KJVb9Gd9pm(n|7J9D
z@2?Gqx>{CcSYeAe%k1*^b!tCbS;9B{`YzRab950Q!J3~|;~Ag_Y+7!5Zjk1vjiSmM
zBue#Lmr&I}!bZTo(Y8K=-3bo@EM#T(g${l9b+FwgMw4zD8}ipUGxL*<#7>uZP<r%e
zr9#o_1ithKGj4ZQWNjPn^YPQCVRQUFMC%iF<z_f?dth|UFI`C2HvaeO@vBJ$S73#o
z{mR|PE|(}PRjv1mXc4GAa}5a%`n)R>U!>G3$4-uIY<2gG^{gJfM}0DLzCZrGLZQ)a
zw27zc<=&*+2?TC14rIFXVWZlxQS15L6@LE|lJ-U;_ya`>vt5-MIvzsz{vp|MX`{>P
z$viE2YNHB&n$dZRp+CrGdc}D#oMv@Azs20fub@RBLV-mw3R*Ki`lr4POUcC+ALbkX
zfO*4Nq2_HI)Q`~n+`Fqx>UuPNc`kEUO`opK?wZR^J!LC>(5RZQt($G{r0~iXn)!^j
znS4vgDmF8%;Msehoddw#xt}YigjUvMwxevJV+_Wv2&mR2rIG|s&#qes4XWF=U8|3f
zB*v-6m!tZdheCo^;g=xv7}0|B=LY-is$GwFe^Z}u5pZi<nm<<E1l`_P^Q(8&F}sGV
zgcMD{TGZ(~+M@JT<!`~PMzboV7kVzS#FPqnDQ@-SyG6S=P)B;@Df|=d-H?syDcDig
z<4<Yh7xoUeGLaG=+-pdO`esw2MX3_$(G9b8+Q|9s&BdAELGVhvw4UbtaYq@Z`}5iW
zYwecMAKR|ev?DVs(KFwb3wd;m-ZA=>Ul}vAXNqQRd?Hu1-R>wheTUai1+N08SBuKu
z`r47*?xopI-iQOy@A)!!U9lKqv!u~UvE@y+8_%UO$<`M7#A(wiB9C6GOf*e2u_z14
z2oy1t*q*hjA90|kkXb0%6m6KWgJvo1L;xD`gN`8wRgN9QEk5wuEyL!=GG7+4cO7QK
z|LWv76TYPzuIdL<zPOWmJ&kYs(A@itysy0_b<$Omqr0+=<{28w>GbBq+Q@i8ctDP;
z2ePCdVSdXnXPHpM+bfgZni~3vdq-~X9ux%Z2&-C6N|ujRaRJLX2sUBB9`+zT*zSCg
z7j7g0mC;Pt91-diL8WI5x9Mn{DLCGh<+Cme^Q!e)-@=~?OG*(BF>Q@G|Mb4^_GBl#
z3d6JbsuR&*J$k4p*tZ&MYAy3swf`x?k!hb!tW|nJU2eZ<|0c5@iCCFltDSmpe$Css
z(yIj@{=IrM%C<qJ$$BEAZz$SuMp+vfvLeMsiHbNp5w7m@3XQi9DaH2QX}XG(32+P6
zCT>Du$ym}sIQGJJ#$P1l=L78;SJe4EuIxJPxsS>s0(KRGMO8F4$5A8XFsp_ZF>az-
z8v~WwJJKqBfo;l>McXBonO{ufaP~VaWXQu^nPaLtw})>J=-aPXOvqGBVxfrzsC0-p
z{rHyEd0LrwPIl)_I@VzzzJUu}&ErA^cUxWEa9LXw^fL*&^4&>F;f<lUaSuNqWb{fN
z!8^w5HIu)u_=cN~++i_02zoL%L5_JQY~+rcRKz=JjC*a*SH+_wTU(GR_@LtmKLkAZ
zdbCm^uNEITK-WNcfe0$*v~t(($<HO@eGOcG<fsKSJNOc9E%7_<;m4ySx2JUPJDQB!
z)!(0o?3DesClZ6L@^$_K60y(h`N*`~T*&J)r0=dN^nWbFKhIIILW;MI?%chuJrO~#
z+^JPAEFnk~^kIi~7+pez+3rj?NYcdo#ddiUJCL#hg6l`@hUdHLnxvw_$yQV;c~#Jv
z+shf}(5}$5v3};wd70FAv7nn3$ichqCnmhJv8GtP9EbWZ@wR@(b+iiYv|?vaP8#G{
zW$x8ff^ST=pwPpR#QhN|rVS^a+fu&>+V5NC`tB3!Qq4NYhtEOx;aZJqtfb_60`lUE
z8BV5PInNIsZ7wSY)8yNu4f<@|rV6WV*&w?ltmP*hMnBN%Hm0pzW3W<UHp{HpQ*L&-
z59X8fdK=N}v1xK9El;q5GS@JgweImA@`gG3qBTrG!x(JTOr1QA$Tz!HMa5EW3}Qnf
zt#NK5(c(K^ArGDgxlg+tKN)PoopW%{T%?m*|CK@orE>Wg+C0s>y8&b~<(}=~+1-#C
zvF0CI9ii+SzOHP5t|c&7{oBK15{^OWpsX7p6Bt)DBKy-!2Gd%hf=rgSnErUns7KJX
z0_&-jKrFEF6H@niUe9<(Zd_cENJ{vPf`#h(>g0L-$X;H>@UkB^UZjr~tNDn0pBih(
zAgNeh8dm3~3Vwe6s4wZ-$yTl&Zt1QDn4%u|;OpfbZMCLhOA{wVu%SwoD->qQ_`~0^
z0$SqKPL@k&S_JHd1i0yaHgb8OEk0xk9n1RO^fmZML1)pMSr0cucbozE;I#AMWbtgt
z^`5Q>#)MVx+|(KMS)+&S1;<jXs_bg@_-ti>x}M6Om%|Q@fM0>VQsY7w&eo#Z%O$b7
zX~y&t#%Kt&1y$t)vU=L<dyK@tQWN&YkYLVQ>lxKaG)!Sl1jRXZj)v{NsStM7^;J~J
z6OB+@oH%jvk=s$@esj53aMuvOd?B*U*#{S;vbBQ-Cj++Ik4{DpCBFT7O~CM|PXdU(
zVtM4SM-8Ryyr#MySg!)bi|THOHn4rwBLlkPqFuKSdOoXLIVGx2b7lW<m>K0P209%T
zne^caFA52_%?cYTHscG}efU7h@b~6EyCUYkl0Z`ltL`hb`b%*k$><L&)AC;yxBCw6
z**xyl<32PO#HHm1o_MlLxAbghT1O>g$yzVGdY+3V=Z!t}O)vH_{Ce4D&`;J~gFpdP
zKR?8r&%?4ML8Vm<1hOOW<K&hhpfL(++4IXCrAvB|1_7?bdzWG+*(r+aa<c7pS_UU)
zWGJUHyYaIjaBJMA;02YDP_&TXMb&IAv&9#{l1!_nFHD$pt-6d6n!ZHoJ#N&T3_xR3
z_!$?>1Qmoyns+PZ#C*BPvuhoogY|Y`oyjaFJu0$x$+_`dt4M_2L2fjj>-h^m_1n>g
z={&Sz*TpD=*caxwTsU$`8DDG;%@;WtXi>Gpoe#w4``w1_QRD1OPz(|!Cb=LgBB*}8
z#O2b)mu{UW;|ThbP;T_wuth(9Bc;|_X~u?vj#bYLfu%uaJJ_WjaX?>KD6nW7-#=uf
zT0G4dR#JO2bovfi9usNk7ZnTY|Lm6EBI3*KrPzd2JL-_<+I3TzpVH(_yQSlYt=9L`
zbcf3pHz2#4w-2-1OS%^yxh^tI7WX+(GF-7P4Vt4LGWOiE`vdb%)*;um6dU>G_`GyL
zDf$s#h*aGbgWmy3{D>w?VM4kP#}2kP`xzoD&q^(_UjWmGoPhh@n#=xPyGnp{7n@CG
z$j?s)j4}X0++{^1^m%DrHNc3us;Fl7XNz?oC7poz%k8<AU2_98*jjK7lktJjwq`%{
z<8ArveI=V;=x>7}(xdw=nuktmVSQ{j7v*%s2&(55kQypV<*7$y-Fy+VxVajgUB~(z
zemNsT#NB(o4mu%`_1o2imCMk~zecrkoVZKO1hdUni1l98&YLaLmWv>GrnT>LOiGb_
zsg^rV5Q;-pK>{C)IY5bRCC;UiZN3TO$wf(61b-Z;Tc(I?*qQF1FpCKuyx^Um7jNvR
z8KsEUT1ZG6Uw@R>#HU1Ch_I8^2$K_``b-tEv>wVGtZk5{j~88w_H48R8)Zq}asp>M
zrcV0Yt9DwtU8-K9LC$aYjir|>ewwShymv#$?i8`BJ<{kP@8YA!eU$iXWI#?xkPd}f
zf2r5Qu4+0B5-YsN_d2+7(;Eug2nKEkWQP)1sKc60nHwd1DaaLpY#RXKvpY|9MA7!N
z_n7i-*2DyK6g1kI@Qyg%E8x)OuFj24$gnt1e|kP{0M$7*ccMDu8tAf2673L3%sCIo
zZ*jAj#{=9%WJ`BMR?0>XEVez4w-L6dY<<MYqccqT+(=EV?p+OS@dot9kIwoDoaH_g
zOr^V~phi^OD0Q=6;D%6iep$m58#KTa4;s8&qqYm*09BftU99GN+wNE^dyr!KY-gxD
zlWXyVD}xo^1bEb{l33zO)0Zqi_2|&M*jDu`!v%j}bm<-eJ#&pM!K-{8hZDXjcHzWl
z{F$#?9^Z15!myQ<FAJ=^59RTbpqJ=fIU0s?(MEk>8~(n02<buTgZtDRhaR}^{53vP
z>F{bwa(UGe(#MO?w#GaFH|c6zM(2q}ZcMzkdYw$4DQ;J}bYx%5>KsXbNBU@AQ*y{t
zu^9e^O;s*GcR62|j3g(dYd3e?$0H=G@6I}r`iFXPh(qRQ_pa%s!$y3)c>P|-P~Y8u
zrVUB|@a^SG`T5kAH*I06*Za3#>HwA+pW7rmO0Rm8Ew-E^6Yl*!3X&ZaI1+JdUA@wb
zT;BuoU>0w#4Sc_PaKG@kH7<ocL;4{h9#j6Y!T@3%x+5MQR6V{E7T{a8wg1ZGl&&S0
zG`;DXJp*?gY*9LpLU^?gDlDdq!m&s5g}MdOglv0D0!4sWsFY!XY4;6&ZuAhfwsfNn
zE{VnGx96JmR*Xau$7ivYyfh!Z`*OlQyQG)j{NSw@1-{mwcyz>JPk8{;jaXlk6-TY6
zARjgOy9nuYsGL_K&|1p3^Bn<@Y`olithLt1HE`D@Ot*@9=nX|M$TX^S?nCj%;iZPm
zx8WO&vxxy8+th;5HBwIE<7Mi-+_Me(&G9X)lORKv(w1e`<3`7(=}EPtyicbUzSL?b
zhO26-k1F_>k+w(HZTn_j=Nc?--H`Ha=$X8JfTTmOzo-rgS8f1lH8yP^kG~*Xe0N^#
z_=|+|NWsmXY0C#lGyI+B83U<4=4aOmXeOwsh9?qL+EPx9V{pqAE`17pZRz&g`(5=A
zHKn=g`aWL=N}aaSo;_o_tOb$Z!-fyozk8|(n>ap3e_O_hz_M~Y#8MI)2)L#{FoI)7
z1%xxs8=JAwy~aHc`=U=P;jCm+-z63{j*n-FeA@U1_c%2>Y?YD%37^j90Bh6u0{|oB
z-bz27CZ)qcZOUl$Ao@C3=4BwBWm@mkZPEX}XV_yV)n%cmabk4*Ta9Y2<m-%1vbX2t
zdxGMVg2*7qOi)3@e3myw%<42IHqE!|clI}cmbgYtCp>bpJJs&1_i*BggwE6aEP7SL
zv^qxigF)(3`&%6Tf?mVn4({#SScyU02!mV=IRFzOr5peh>5hZ~@Z{Qob6{MAzY6yv
zWwrH4+YVVT=cybEzOQ7WK>)yt0=gNqHB|w*9q!sd<8pDR+t`&R=U9;59Tnzkvlm{x
z62aK~boIxl;X<`M_p_}l`C$`DS-X=P{cWid1=)igwsrm;HT~`ow*-I(@pS<^s-B-#
zk^siCPX!|3Tql)w=l)x;g0Q;%1R+*XcWP0m`xZnyyJV)T)X%x~caFl}c2HGW&*pGb
zsCWh_Z_&>V?4q3j0+D7K-Fv)Td@m1ejafbeamWw5OO2BT1v8v26aS1)p#BbB=Kcea
zSV20#91i~+LJ#byqLqY|!o&)A^86$T6Wg0ha$~8P!!bw^l2w)f>0#0EMD$(`J5!&S
z?Vw^YTg_SUX1HR&BgQ>?++a&YnLmS)*RLcVrOp^#h4`kw<0l+t%~Uu7LU`KsbbXfs
z_~6aCjoqAV!iMt3hv31Y#&Nbww(;Byp-s3lD%52&?@4w2homyLD6nZ7aF4<`_lQ65
z?CT)an7zQ+zU;Ikl-5+WD7_f>M#weK(v^%7KC#^{Z`DpTd`Rn#5qpwDkYsF-)LgET
zXP(Stf0HCQMNCy@H)EQcy;)gvLt@BBhMyuZ28O-xbGg(QWMB7|3r8IjJc70Uo*}s7
zqewy+u%QOE@sycLpQO7J&Znk(VfnwY7aRMhlv<Bb_3cmLZ+9UJ`5{<U>X%N%@Jo^P
zYaF)g0hS|V`;QMmd7V<2Wg*h;IVp@g_=o=H2VTxJWm?I&Ansoi&GIe+EHRe?hcxm4
z2)Mu8uK|7Mw%eOgKAu)0B)Agdg7cl-nYm#x_xT{7tqz5L$AHn}%Zz0vM=B~Fe19*v
zi1|FS8H?ZDVHR~%hhNbM?Jd$$8onwD-^@5zq`h0iQ5SnV)in_TV0;16VF8lwVPMkh
zU_hXI1ROZ(EZ_jVy#_wQ`S5EGe_(+1#{K~i|9kFZL#-h2jpe!bOf{q3@PK9R#R?t7
zrO4L#kpDEdFT&!tX)XOpWxJyJ&0iDef;Uw4k$LW7>f7;hh3J?K-TLjoKuJKo0JjXB
zqi7tcl}*y@A9QIjN^Se$ya~Pq`<_@jT$BxgD*%ip7qI4^lkXv(!HB65#{Je~)cmJ)
zbN&%=GH`MJvjtr)Nx&cM*#ju`-+S4^*%enihiU`%LCl3~Gx}`Ux2%j)6SbfIY`dr3
z)c71%WW9i-mJGvOinCEpQ!a(;fEa-H*TCG@Ij>(r0wnYRxE;Xv_pyQxI4{mY1M~fX
z*$*}N*$8TIn+p)@&pH45Ml$=Mz>C9b|C*pZ>E}qCj{l`E;MM+a^sn!2VeKeb-Q9tT
zsTO@tL=!D>ZU8+if*1~>pV&$tf+bh-ZX^>^JC~n_F|~3iClQ+$4VW*#sGbW5Khx~_
zrm%fgqtmRaRgoP*j4z#QO(vxN+7f5qnLLY*z1nJ7%@;DTRcR+o%Co1cj*(K~J70%r
zB2jK+1fE^V%R?};4s@xNXFGj!UQs^{sK`G;u$0CKK0gLJD1+c<2l*vWcS}TRS3M@L
zI>&VP#R$oLAwjB#jrUPoD*d*`l>5sJQxk7>cMFqnVvY40YA(lk7Hbm<l6FB+y8ZS^
z)O1=Pvz|5Q?oy59*VugIz4Nv!PP?C=k#%<^F556_r7f_PAE$=aVQB*j%TXtQUlkw;
zw#%=PUs)mN7;#~ZV0PDJrvY+zn0u|yd3d)Y%8;c!q{DV>S}P;?1|0%Pgng+9Y_%jH
z?=s##xIgni^7S+i`mIg%+CKm8R;+1!81t`{n#&6(@md>6NwGxQ>9wInq9rvEWY5Ch
z)Kn7_#B9-eo~x*_)t0Q5r|OmEB>yE0h;)h=wF!)Gf6jGSEGYV3yNIQqWKCp#u|4Qb
ze%Po?ZuGc89F^hpz*|@kruqG=9xoxUeyrvmY$ol-@?aKo$+0o!T|4R)`hibpmOuH9
zTVf$Co5V)XOptEsWZa|l4fm?+jHj~14|CJ#*m%7MIGA$`O5@4D6jUR<T=VN+x+VRc
z#VZGgS<=Uh#Vn(G!xR(W+327b%9CzgiFIh#V6CHhJJZfEpzqbg0)4!A(a1w4{o3~?
zs=A7LRVrD&WfOg0@<c>vlB2ZFDt$1uWz~wC{W6trQv6?XH;vy<fxpVI&FgZ!>v2gg
z(Z0k+cV;T#(~PaGw!94Hl)CgtwOUjC6zG!lR@F3rqciCkG_%|kT&Y8i`}3{DKeq?C
z6_wUt;W77$S8q_wriQS)l4Hpzr%;&ysQ+vKFkZhal$!4YD}+jYFe`|B8bdlvEsNia
z*71vIm-(p#d!RDKc5y^~P@!CZ?X>6uI`PqEW+igXfgY8cmUSA#F6g}rkei?5bx|J!
z&tMQ&YI`k+7A~apcE+@c-N$1NI5$>>(!P{a+b6{R9{#$ONgo7aaTzAro9Ao@mGTv2
zc2uxP{c%N%WMw2Kd1Y;AsZsDk&+q<%E~a1^SneA~dg}sv_--6Ap0uzN;@Spba>s#Y
z3nv@NhEkOq#E9(NdtI^x@<w2yCtqd*6b}L8%x$aLo2mN-+Um&(D@P3fu|4T09$<er
z$@BkkX`$H8*<WGJHiUS)Psa?T_fY16pbE&0#VSDTE35ZFrI||0ycN^{nRMJ@@}S)1
z=2hm0ru+h$H}Jae>^uCab)HPE?~jV6VA8FKt;&LYXAJ+nlm_IFQ8Xe4)us{eDV=q1
z@gyum78F916Lj9aBF0^_!grV{^sj@F8589WJ+Bk>ADeU?j6Ns257W8e{T#r>|2720
zE=65Am!rumVFv%~<p^Lp|7|b<n>qGAWi@KTZsDG;anJ4FvNwKNUP&&$N`W@U*N3rb
zq9#(6|1q_ks027-&Mp6Gg#mNLF`==UKke^-KBYMyb<m2J>3kv5CKXxzA72L8c_Nnr
zh%yf1<1Fc?SNM;i|K-3q&W>aDIVS$UnM!~&+fQc=82JCGJK#zH>kW8PTqpP9RM?oX
z8?7hMunZY~^Myj6j!q<FJc`mv$abh+`4QZ@oTN~=sDyv5N8g@}kDq%`*PHZ|IDRmI
zpaq%e?_4>%22T@)4Q9?>_FEY8?xGHdxgfu&ds%*2K`Xs}X-7%H?)!V!z=BDFP2a^P
z9x;}%dP4+v!ra><r@zh3x`mFb=Q77u9a;$z?x9YtsVUl4iJKP_PLU=H3yk&7nfOp%
zsC$~X;)_zp>~6we(szoOOHXs<R-4Mq`b;{XT?h?vZ><f5UH1I`p?oz(^F0!p-&w$7
z8WG&;xqiz1moWSsdC~I5<kk8_p{e8$cS#i+JCC}SceW9sqqU(Rcybq+&0H2arFG96
z^_TH?=?geTj8?^iFK+E9?a$e2C=_LvKjrG&%emL5Q}qE4%JGZK85r{CSl|@6&~A@c
z@sxo6vq<(mPu%?1-}oPU^xmf784L%aEO%<wqF1FRVg$PtB0Aus_mIjk(Ak0B>mT(p
zYW$Qdyjw+mb)UUsq?4q87tO%17$Ic(f=l4}7cI-~og_c$2_IcFRdmI_LlejE_pmEB
z)@?UFQ=gA76Km?J2PNVIo~tV#Q!s&FIpj=m$qQM+YL$=_QqMt7R@X^(`7FC21}$=y
zH&)Xb_LR{m;xQ-=zIfFh?nq#=eB55g_$9bkH7o|d46WerfHwMY3^`8$@%QtCh@q4p
ztshyB=e5{N7dz|0r5ol<e;~>MqJ@FrSZc9dr-)&X6C)<PkK>#tW^V0%ZAR(`2SQHn
zMt5Zn8!!*;81HP5pSToOzkmlM--=<mh}5@K@)4qB8Q*_Q6(S@SB$jD^j%nUT0-EYf
zY?pT$(3bIdCvQo9ugI1WM7?Cdq>niVbR?_9+Dc)PV-kC2rn{$*pc4C%kEk3-Mmu06
zQ1`Z>&G@^186_T}=lGzcJ_xDwMUY`@eoEL}n&ev3{1)kVN~*NsP0_==+OW+z@AYi_
z(bWmw#ewY&UOn#+Jb5+kx5H<+N@C4npYIBV<BaY)^hPc^p0|s$qKsUCaid_$)KAL1
z%Y&g%<`Tks*{Bn%?a%`7Vv%hOo7uvckaKMg1Is!}ufELhEF~W~wfa|cbGXEUrda4A
zMVJQ-ViFAvme*iI4P+z@9Xxag0xP2T#_T<P0_WRAZ3=AE)_HZtW!GL~V9(z+V%kV>
z1OjCZ&0XwQa?pW1lD4ZRaKeB=brWkGKdW`=Ras%)N~;pviAkHqy`Fp(?i4j`3bPj9
znIg?)`yn5t#`HWe03=F|iUZwsLftKXl@9SV<@O_|-Kfm=&M_v#A0P<sr!*Tf{mI)o
zABwrr#a*c241-ff8K?=+pF~y2KAAw-W`qve!S;K?0W*1=PFID9!}*D@RIQuaH6QC@
zU1o36R~ED(%fpU`iJn~!>WiTCp-7MgtSv?IY~JQOrDMhFM=d~FaOS4LLj;{Jg|<EG
zQwMjEC|_2c&q^IXsD#P5sw0`v>#~}^stmdg4<X`@{uuCssikjbMk`@t-!LA$bHu}2
z>MIfSTAxfCO~=Z?2=X2GnB9in{{5GZhT$se;>}NEewwz$iNTn^#Di2WA00N01#xHd
zt8abwUXKtFE)IW>Y3e^)X7zZZt%;Y1($m5!xI(6NKP}bIhG^+j-4$0`2wq#KWMSm(
z{W`5)oJ3~JzvP61&Ax2dnkvG*EMNU>3dq~yp3Gl@-lyC<FZO2^ue`RK*t*itM$-XY
z9mj~$dC+d(nNf~rV-q9CprC1+OUDndQkj{?TfWZ;F+*8p+-MOEgLbc3xtgc@<6yQb
zN)})ld&pOi&C}<(w?}I%vZ4(>Cuiz^LJf=hzEB-%9Evo_csszaOhSN!H<ve9E9j>s
z4e7Vz;XcH!3vBqs(M{$h6B^Mvzic-etb+=~RmN(aBrP$Ac+n`V4?WE{2ma=Uds1vi
zEz8Rac>sP<5b=I<RQXM>;GpF(l7&iv@b=}QJ6Ru0e6QAR<k(E*e0JL287{5dT8JCg
zAIs^=?oBOa(ffQgvd5Y)<+gdd^~?atUOxz!O6I3Y2*7X95&eYOlK!_it#(w@b(wqz
znapTT<haxF-A*r1rc^e86ZGC&`q1~5nQrgPn%xc2J(rF!{?Q}?5k+Qa%^wHeZ+2HJ
z9x`ClJ}G=d;@T21v<fIe&<hkBVvZxC8U!6RW0qq+cg6!=%x_iL0j=sBa`wE#JLNat
z$=LYJYU_QMb8NE&ymdqm=gD9u{p!bil}^6TYj`5!Aaq;o7zSD=`0<XrY^hWFJs)?C
zcvM)^ZCg1I!ot$5Qmtb6l9(!b+5QABczE4*d9crE)@RkJMDIbzE$5i6sCa72SRiH!
ztm917RA46`VSXxx&m2vx$4TxSD$Nh9nIf|X@%#Hl4&<N1gs2o8Q7I78unUbeE+xLk
z^e&lC?rx=@Ry{1CrMS?0GIGhQ=uN^Q^!(DrAIQSJQLB!1qiwt52D>RlXZEB7Taxkl
zK7XTVBoL6%(G<cTRZ06dZb|4|Ii{QVaz&E}FJ!Br9^_)<@6ILckAm{`b@QTQhxn)3
z5R+p;)AqA}K*u7ofpm9O3{GNsH4C|03Ua}<o#MnoQ6*UNorxd&#^hc+E<fLn`k2ja
zM!sOi7t*7_4>DgK?=%IH_vVA1Y$^z>gi*chRNb#`a@eAIAQl9&ZBu+KF@uJFBtzEG
z2<s3>rXmhfi<$_j0&amBle)a2_o#zu029;M>u|h9si`OqWP@{{50gd2X(f-i(Xv-f
zHRW0kj_~@0TUAwm2ZC8zTtW(ZxcJm?d?8v8vth~HP2}_OG`_Isg7ovW-U=+@-tP4v
z1AD=WC_{xxtf{3;$}x-F?+M5Kkf=~!%EQB2QdvgmkG@PH{2k|(SYnpn^E)FAEm=A*
z<X_5SeC}3lWXA$gonN)EL(9uygGlmHUGt;iJ0^+g=Lhdt>p)Wz?JhzyFT{TH?(l^h
zoPSXXejw{}IjF)O91>Gl+{atItZW9hz-~J0yckK&g$Y?Y$h@*4TD6~jb4X?^e(xf<
zgybi9nzfYh+WEjpubULQA$Hj=4<-ik9*qC!%Y`Yfsfz`%%K9?*_Xk#Q*nvhTrhIM7
zC%XadiJG8pXY5G;qG~*Cw4NIk?OxuqphQX4Zkasv7unF=c20}9$~Y8{1i{k_5j6M9
ze>>Qk_U0)|&3E+p-15*NEzB{K;`rHx0_((=vZm|R>38C*tAWL@IImE?qRW8<@vI2m
zK`%)q_5|nXaadCad*ahSjIVCCXqg&_FkgK#b-=k2)!mjB*#>T(qM*2s*>BB6wlns2
z#lF3Ev#{&VJ#!_K3^`a4%jDB#bFk2HlLr6i{w3}55&c2Ze`vVpC&*_sJvPbqm+-A4
z-&UR|P0Z~{^AiT+%&P06&coUzG<B$%vF1z=#k8_<k{k;0O=ehin-A2Rch=L+sxoVF
zR;wip>*{_)+l}~k6y0KR;HK~ZfNzi|+K|3hTjKM|#Y(!En`cdk%)-9Mi8v7l=7Vdj
zB5D%;_z#d(i-gOvJz*}mj8?u(+r~w~h>eWLK4R30m=zT0J`DVN+#Md@9)$&%5P+tb
z2SfP`aDY<Q&e2OG+fg5m2|0k&6ix|ZVD}`@)fv9eu_=z`U}_x-&&FjCgYm)K0Ifj_
zfT}=0$Kl7Ua)Rlm@tKZ^$tbsmd_LADaCwR#$YAA`1#xiH{-lP?bJGuU++|MDG$@TR
zS)L{BzU@#!Ig^aog)P;mcXF}0hqj}l^la=1#+EGBF~%4@oAHib@wT(EaU`<qnfzdb
zD@zLc^v3qbYSdSaD%xrFx>q$2taJ#8P8(+|R8g;2J&rQ#7IqNJs0`*&c*=0gG@l6N
zbD=5gC#2%~&HEUIZ|R^du0PIaAEK!G)`2X~NA8ALB6KFFd^blok$P?X`8I9<$@7zD
zNP!>rQx|ryxF_9om4mBJ#8RWOu)Z0xCTLmX^>!3|9&g_U(Sx^P7uX;^71FuCM9Jrz
zi09g!=*5vxoJo$o59zyA$ib7xtrj?uZW{#LX6)Vm@=hoj!*pj&T_2!%WZEqEp`s??
ze?p8|>cu@TC|L_Z<XBrliQV!nn*Oq_*g(9!OVkje3rzO^R6;`YEu>zC4v6qU)_2Z*
zz-ufK{6CN#!}K$)scucHI7X*-7}Pxn>r5zy3HAQv>MusMk(rGL5H0|X0B0M&C0~Mi
zohuApKWj3y-<&;)np~7AB}0Uywr8>OVdvb$R*fjJeNevCDf%oOnz682hclX(OV@Pj
zD+Lf038H|>7Y1DgEnl*!J#iPbc24H~aw3S`Iy8#?Z2b&<Q57>1FXJL(+ui>0W@RU@
zpUMH~fuH183*;sjpeRz9O8KbfVOzb6DpwtauXUj$#3?7z5BY7O;tLiE?D#H{`g~GH
zD&CjFRZosID!rIK_7j?N@PZ5wl_g=b(iDL7HL7go#L`D-1*&x6b#p4o^HpWZ7A=G(
zKq?T$Q6>@||J?5{oW{u*{W1#xG6k46gI`47{}@q_!-)QepaVeRKdk{|!kO`ZdCtM7
zoMAZ6ekP=Tb+8R?Cj;aYdBV(gsz2%p)<L~xxphNT`%~<Yy=c>mgTJPjZ$+K|7j5`A
z^#?#x&Wpou{lrF`z5dN-a`300aNRC~*_L@LLsZxItk~;+2D=XSJ0N~Qmn#*wmqUX7
z9OD15T@HK6&0$XgRB8VZOFDJuk6_4V##|eRiueaBgHB`YVNs4MNB`CM7xDS4&3)hu
ze-8BDiRRzX{nuQ7cPDXxFJsLk+kk`Hv}(hHQd?ff0044sK5BLOGaL`#(9JEkuy_4^
zK9tt_$dF&C^&9gK1t<8RQ^a~fRH2UA<$F;RbGt(M5#qq$95DXt6waqltBoB!6@6pT
z%k@<72oxqbJfY}()a?$*=@g=XgU_QaPc>O}J;%`-6snBJJjrhp<Jjc}Q0m_pe6t@;
zn}614T~gcJ-K%M2Rz(p1H~tOdT7*-nYX;k|^y1&@P}Y&Vb4LAthJ>s&o0i~|-`b{I
zLt@t6<xBbb&iWJ?l?8APjLHMekDSS=c~N2jni-eIn9cU3(v~K4Rz&{IaEwV~G6p0M
zo)myV@U27*zCVcuQ6+_0G8m2c@8u%`W?~4m<g3iXK0TtfF8*y@xwk<BqDi9w#C^h8
zd422LrVni3!t_m8t*RHUY$#GpoI;|NEX$o7_k&x{XYbX3M8)&EP>2Zt=gDWIsB*7$
zB*H1igSuU;gt)j+*KxRits`FYCoZ*>Y1g<#`ZCACrt6t?eNQ-qX!&GVU;Xw7hv}`~
zshOQZ2JXyP7XoJA1UjZPtlj@@#Ic7^sYhW^1KmDX1sef2B3HlxlwRK_=Gpt{%S>Ti
zUrJ+*JX&8@ccC#I?A_GmOSZOMHu_fPk+E0Ib5UkKBQW}OmSI$-zH>+Bp#F%o*=#`O
z1%r&q>8O7Gn30<-uT*u}EGxf<%SJ?(eB{amU&3$5D>wC{T>%_ac=5E+F&bLc<X)?E
z#Z4cW!{+q;c(d#lcCUKyH;>5+Fr{#EDVT{#TA>dDq{$X!8YqcNsSeHibXEf|zs}Y=
zMA*~shuIx60wA60iZUgE#2k(Vd~9c<oeop-3psQ`i}r|8sZCTb&!4=o3h*+2kl~GE
zJj{OU@QK-^toh(uEKYs^u&#X~4kino+K4;nB0px!StQ1vjP@P^48pPEx`C5--}d*n
zQdSdV3Zgc?aY*j9zHaeIc`_#grI_4SYFckT?6>iKLotcM0%(0sJhpm_<YZUYkk*f3
zy0i^t2|5Tho#xjlx1JvG!=*M&umW~kQoQzgWZawHp6%Rs`qgJu)VK*=TEs!!T*$#B
z>9$Fxjmh&Y4n6grGIu2-Te<UbqK7fa`spWYu#AkL-MdB$X7(nqStRjy&<zW=jB$pw
zm~AC1_d^ArU{_JX;q!iNy&za)u-djmlytwtMck`TWbH~gxw@hMZkyVIoJk48dDgWz
z?S@|wj?zF}la3O+N{!2ZN?RuE?+?`ojH;HQEhuy~jBQ|s7g?4<4*6wp$(ML*4397D
za|10dE~TvR^LHiq+41+<6O2o`l)1W61hx+~K2^`7{)>4d%3zY*8$zeDz~C}>L`d?q
z;B|LNYWkPluPe$wP9K2sxw|j4O5>b5u{<#Fbs{3+$rbHmfj_PMm;s_jG9OB<e1<LM
z<IajTk>2r>-aCn~+TJX4@xN7g>sy8_CZigw;mN_4o7+%)#wl*sT&!vBy{kg{U1@wm
zw02*?z;=)HGZdQaJR5%_@oATP1y#@jcmAXz8ebpmNQmutPOT%mV)&y=>R*^->)H%!
zwISREukDTwMJ-PR!fl`+$F{NzW4O~dy5tFdg>4#s@v6lmxc#4vsv};b{K=$Q=S;9r
zG+a^uAqkx=3SKgVj@eH+rCaIoc_qFmey^;FJfTv+ouAnk10yF@L&7bF@^hl+cf83l
zyO`2oPF444GKuJ4<=Tp$iNmcj!%_}we1GJ<5T=wG;8Kgyiv^|c<VQ`Gto4aDw9Jk(
z-)B0qTHMFj{chb7iih5u3K=*XG<~l#pJ7<`Li-{or2-D`YuRE39iaXwN=<Ae`h`FQ
zofFFd0FqZOesAu`nubBA^Oqmit(NQP37kabV30j`LvS0AaeCb>-_LO*u_`C<Cm`R;
zdY+LXQ0l<EM>|a+y}u7t72oMtLMoQF0Czw!&VjRXKBg_JLuSZq!Tf|?BI+UP^QgW#
z1RP>6_8!For)aK!2yfEjd58MIa1_3dF3mk7>=Fo<KAXh_sHMwYkJ$jd%)$6&vMS0M
zl{?>Omwt~N<uIT2^~ZCP1g%9rN5kqVY+^$7klX4L9SQUyinWIlMq~Jga+x={hJ!{K
z8!8bb^-&Q~?F^q~v{)Rj>b-K|`(RzKkCyyh^`hCtocyyp5;}L^zMm}WG#Ja#OJ(_A
z-b#b2<ZJ!Tw!3^|mF24nlt~wTe+<OIr^3FRWj9V7)H%uvk!fdC^xwMKWhb2|ic*+w
zy){n`&*RW_X0O_2o_3@r<3oG54?r_9zAos;3?8%y3ERF92Vy>3A~z5AzBp=z;`WQ~
z!Tng-qPrm2d0Sy70lD5l%E_-<#6c`8mwH76ostua1A%3De1ztnOPl!D#+oo|UKM8Y
zp!e54`y6NcE#ZJ)c{#{tFct(eO&)7Rqe>e<)8K*f;kg%>LO1`mSckhZf08bB)!pu$
zK<3JP$HN(;-<2MN)HR*KL`9qtlb<;(zbLx!BGl3WN^8%h<<z};H_e64ucii)g9h0;
zf2G~nbHn(8q?;oU_b)#c`J|V6^4R%SFOPS^4odh8tHeSP`#fWlOK4v-$pz%l9s$r9
z^&<_oP@?tC^NmTULMXvmmYamP&p&Tu&PnV+E#$*dkcxd-s^`GMMxb43OT=-{GyM30
zOyH89LubU(-{(sEjG&6oUDZiUJrCfwGJvSODm~O_0odufSI<VUH@F|Cz^bX=ib5iv
zAO$|gZ<^kb@o!dA#ym&a`gYtmt88Q@cX0o?i5O{G)V9UR_#?^w6yrmK`j(lNVkuwx
z+kG!p^kvFY*7Y6{jhdb%;x|j;;t#lhNW5riJMKP%?xg~Em>4Y%<Q1YwI}pn_y&PoX
zLk|LYG!Ca5($2us+muALJYCVgx1tjEGrebS2sy#Tc>J~)rf~r3HV({<_H`%A0Rf8?
zMp*wr2~RmH3E_=Z{3^5Gc~)rzwEXPys8)LJZKInaJm_rKJnJr0CQ6+AT}##^1Hzpk
za#^g_Uue-@9)lt|g*ZwT8Q%;(@l-LV*bGb*S5kuvVIGv@FI*oWgvOL1vhMimKw_8h
zpz{j2aZfKGbx1C@RW)~_RKNut{g7kbKFf7UrQ^R*QW;=4OltT<I|@I1iCRSdHayRJ
z2(G=OZ?xb?O@RO2hV?BE@3pX=HaZAy!HlsBJQuB)H?Oa@#qW~6_Y(Mof1ySQ)I#hS
zCA}TRys5`Mq=g15*Wy<J?m`eWmW_IaZ?L3)@O1$>|2%5gt)+gY42iDtVukEae~DS1
zleYOB>Rh&vwf|b}VjN%z$YpLzX2$+VK_R|}Jxx)^WRl28l21%V?wz{@fGFcH`ymC*
zO>#+#txE;(E}u;oh_ta3QmGtd``4afp(mDAd{c|fyNAS|#^f2i9*WG*CXE(3`#aoZ
zq48_A<uo4`1NshG3(fAo>YHkzQB42Dd#l1EJ6b1O0i(J%D=VQ-GxGx4O~WeB+5|>V
zJ-Xg5e`Gl7W}hKwvo32?<ab#Nxw^M+MUGaB(H_>IX7iPiRG}UcKcmGo@^hIh7T`c2
zB^M6<!C%ERIhPGE48mZokA~l#%VI<W&Y4i@SOnW$?rmyEP0k~g#p;zxTolJ{3PWMH
z>{Xs!m}_?j*oBcho=)y?;t!C44qAT3t3`!7>%$1^1x>?k#|-T6U7o$SD*;kAMPa4B
znj6td(NJqseR08bu>)EUsJn}%k;#*;wB=4mrZo6fu{VLerr6+)vtec&Pf%tsXyw)I
zl5YVp2}Aus*2>Ztt`Fv=$c@(J1hKw5E!Zuab#Cf;Rq`jjW2je3)GJynY>Fo!lk5BP
z$j9-+k#6KmAnVLK1X)a^1Sp-TIvK%Ga;t~H3qlZ1NWNd8ZDc9d_km2wkSYiK_8mTV
zq-}Fyd#%{m;X0ptwZ}tkRp|nm5uF53|65aXs;tj?AJh%^>TJRGu!K>!c`k@Z?M&NW
z2Fk7jREWb&k$NA?r%h^MRL!qEYfVA)Vo+hU=b}1`K{bh`D4C4gppG_cCno0Zn9I#}
zXV~nMzY}L}N*6g$<p^mA`c^dlY|ZC8Ae|jtUExP${H|}|8PG|UO}h^3I@uf)z)1~i
z<>H34|B1|h#t=tHiiz=FrpIN_%`ul-)f$vbdZnFgK($%76)WLRdQE=nqRn}<Iq82o
zadL|PNm2fjXjEMPSy}S`7H|Rx_;1YlH(sqc0bSmm)LLGZsaWGcIP?pC#Mj>f*P*h-
zr2`+Pul@u_rpB`%*v}f5UjUJ_sDJ7>I6xQp`%i)sc>fEd{_hH5q%QqW#Vr53M*m&4
z3<tgcZwg^JxbVMx|NqgL1{ni)r>tUx-R@3@1SP>5fus))Wem47Sx8|6S+=*7p0<dK
za1KfEcUcdt+oUWdm04yl!O3#8hn9Q~MgA#~*r>TPcWdqzaJs-BXQiB+a;&iLX0A+E
zmrO%mmcLrd+t7|N?Sr4&e#z0VujgBQ!p##PU{(Mv@lt%k@6^D4T7woKQ&I?2rQZ*|
zHIdbGrp8UL)~$kndw*k}MzhG@s2`A#w?bZq`PrJW_S-GOG7VyjH1)KV1OJC}`F|$z
zX{A5@+57)3E%-GXNnT<!YIS&MJ~l14>vzT3pJW-QEXP?glKSQ#x1>sjKJb&)pBn#D
zYyajP{!JqQYIhy1c>ypFAC~por2m&3zVMfE(24&(hYzg!R~r8x+TzzIfC2f1#{R?o
zaMbQ6HNxot&;=Y7`?)aS6Zv#36`&rTWVyz`Hz@|m0^I+F3JN7)6&aKKMZYaBIA1Lf
z>tnNuex~?6mdQ!<Fn2ioUN^=m|1LC!^-d&gPFv~9A|{-`Vp6;P7t;@(IvQ96^Rs!5
z;jg{>37QZxzddIrbc(*#va+y_-<s2A>bESl1~1esw{9x^tIB|+C_v(XOoULXgz7nG
zjTl7qEB0T@jYd!LTkp#I4r}U{MKcCr&3jf4HEFWm3QYwsopWMtry_=YS*MdlI$58^
zVnOb|7f%BKI5HFKX`=BrkUEVaxH9!O0c<H!IH9?IHA{yUJ@li+>r!eP0IxzN^k&<9
z_3Y5o98?;j4#X&=7g*Ex9ERcE%=3!|9<=xz<)~O}Xeox1<iAIr>#roa-=mH82N{Wv
zD#YT-URZ_kF5;t>j!3wlKJ<%Gs{nq}<Mq|Z`HAQv{S~{v#IY}MqlR#o)^uQv#b{J~
zWT_Dnef=D@MTZ7J<N34(am{t*$}9<>l&xRJ5t0BHQ=AfJP}cG>I-yU|C9A22Q)AHm
z0)YUSpl`g9eA`rtH|v>lueZ&spP;;Y(3;Za&ak~YWV?9y)FtVvFFr1mPBXM)C-d{7
z|D%;5<v#z5=8hDK^(_{f)yG;1n}ShA6AiE21}|D%+gyb?y7m|H$@*mk=OcSN2y9Lv
z0{sXN`nJ22AJGxlMepXM(37w@b$_c0Y7+Tdg74x^NCA7N-lB$f*4dNUFLLyjVanrG
z$ADz4?}SFFd|HD}BA#4CMGY;s(W-5bht<E{Z*N*0L48@okrT9UA-E!PvAww(xX+et
zmhf4fMlurbx&;&ryrs;+Z8Vo#mx9Z+8~T#jAWBEyb6VwayxzmVd3p_K56ap>3IkdA
znY(_m>MFPlWLH0Z^j80NT3yA&4FG97bt$J4>z6qNLa+Bxjue)Bdafb@Sh1)wtm$j%
z+wU6EqvyW4i%^@xty*q0CP&BER8Lss;x$73qt}$q_G>F{zgOca^kQDPz6H>B?<nOc
z^aPx$HoF2P637Yq#`LdMcq$3(P;Y*-5>~GyWU*}5Io#HS=g3f^&Ec+y!%JK*FB&n7
zkcM>!HlHmh{z&!)$O$}AN%fQS0GdVF&UUVz?9wQeHlKY3B`?^WA6%7P4uv?EA}&<a
z(R^|c+2uZBZF0?Wj;k$+xzs;<PoIY{>hsAFqiTc`$fu|@=Tf@;A~o;~d%-Ts{hs+L
zmwlBDi!YY8G=?ujaec`t6&?#>CRKqC$9aL$fyq8TzBN3NAn2{h&lgNbUgSiJ-N$yF
zNKxlrQU4`c){Wi0{ShKDgv{3sAh3lqH4=?5)gWB;F?D@+b&|KtK<D*}TpWx_Zc<Z9
z$wo(_g8P5wzzLSj*PQfsU*{8sYxGrE(eQ%1kjtqJigNX%MjHd#dn6lm=g{&LP`22)
z7;>207|&!h6ok$cdjpj`XP5XnXOoFtC?pKUYP|aNOptWP82kOY)RDv0_Ml#}luv(O
zhAIIF?6{rJU>I%#w<(KWF$h$oX>3Q5ACNowiwS*$y2fG007^_sM#DF6qEY(OewV^x
z9&GivNt*QQX-syj$)yg-FB<$$ecK=zD~;)MOqz*h2dtg-45(OhtfaQQReJI4Bg4rS
zMR$SKP#5%sj854xxx5M;;)dkcAz-2osyPAVlGqsdP`i_{F7?9vQ;LVXG2j<jRZ24E
zA6Z_bcmZr*fE@2)sc0V~TGaPy^0bUf0eYR2F3;$TP3fH_sKVYkDz(ZkaI4Qo@fP~*
zvg&esV-&&1dLZ>xrONbiIUr+>lQ3BfkQmWo&^Hw2A8~%O^`Lxs0&wG7;-9tX<7y1D
z`fO)6hZm~XLuv<w3>8~<W_>ojCQ}!=%t@A;mGE(s`Z#r=He-7a9*Z63=Vo&XL#`)?
zAL*`^FUhc?STa9f@HjP<dmy<GdO9W0rb4BOq`h+&2>zB;+EF~9rgjuB8=wq9JZ<2}
z$;}Lp%zkT(IbtA|7#KKPwk)I3^o%Lcp&Aa&{KQyz89Tl*L4{>RS7cO7z=x2s77k+|
z83%aL1iYjcU}oFp|0Jp#7c4`Z7k}e^9{({uRd#*3TSka0ZdTr8P@7^4<x6n$+hqqA
zvoH1!&l+~t(yfW+`C;MO4w8IvM0_lE0iCK(BEKAFnpW_&Gynt!QG|#6+5y+l)!N=y
zD<S%&tKHdgO8qJl$g9`nOKNDHvy1u@+dF?~^nc{lxwo@-L!y5%8F0G9NxqRNLT<0+
z>O{$0b`S13LSo5X04ScMrkzenDY5PE6Fq!Ry*@D+BjuY8&gC-YktuZ2d%)7dOs3rs
za^O3WwV#QqcpLuVO~NtSt6St|4rLL}v<9ohI^9H@o&y{SO&&^&Yf@w+Yc(4Z)lERT
zOPQxJ4mP}UpV!)bbQ0*P<TdlOkHQX>c2s4Hsw8f-#4(KId;IzgMOyt*|DmZ86G7?g
z>vSqZvhy7YEq?||Qa6qTZG$}2mQt}dy)#N_qt>1v#lc>pJUR1AhjR)Rb;*ZrxVST<
z7!|mTAFJQsPRLwrvz72LDP;7>SIbU2&H5olxV?d&tbY?jWU)TqV6MkWzS#qi8oJF@
z?HZAvUWd<BjtMy@Vok5egl1`GRl#bW#LxPESe8qW5TWk=C<vks0tK|j6+mr(DGG5o
z)d%*2iIx0_ha<LXX&xLy5{nxK)F!{yb8mxD^Yf#(xzSxNxmZuo2Xp*Rgt6dYJe)k`
zDSyO)rP$Af4tTXEuGX{T0jO!C;~_jxT5Ix$-J%j&&?RJm-;pHX!6&BZ)y5Y@?MRUU
z;o%GPJK<IGwRm`j9$J@CUJaW#bsm#pM{SD**J7BaEM|dfQ|2na62{@;;ncGi8=20_
zCwGE&ysT2EYv@|h=Q0@ASiZADJZK3~e<6)u-2cFKd#;L)O;kL<Tg+AOw+1<YR8{n^
zV6Y92V3k8)sU`ZWC<m3H*S1jH<U5VSoMRP*;&@Q|rH-1qy)`S4Hc%BqgO>asoYMI|
z?EamSW~H+UrmH*SY;C=s{+1J6&Lfia;O%F}S$ScH#D=$BC@*qqJQ~O?w>hw0EtFQr
z1L;!9PweetK?Q2&SO-cqfu4c_et=m*THI2_kfgS006wc)knlswPrtICAGglWMhcJM
z|KM7ME}wzoxKoA<VS)7N+%tYTiKY3C-(A}aROd6Y6cFCE`E`6M!)sIsGZdvm3~gIY
zLO389M-F7T=7AN=z{_>sdKrI*D=D4iqfeW=X5EHvUa1myERyU1{)+=X^}}%>y$GC@
z393?adNXmY4Fv!yF;NV(gA+pf#exK1CWkLO4!lIlWM^1**<DdjAyH229d-LkNwZTb
z7d$I;X<4uKhMvN5x*Gnx>SOzEnVT916ZOwL$>c(lOIbcT495)r)LUJG4sC{%C>zVD
z1`N4s0|)(!+{kE;HL)$HY83omw7my3oa@&<%xFOn1kn>UdPMIMq6X2%L<k~!5M)Fz
zsk$U;^oVXSYIH&nJ=*9cL>(f^5XSI5Gm@Nhe&zjt@3;QnTC>g?XP&ud-)-O5-q*G7
z=*(oz7A2WGHl!x(x$b+JrjPm804bb+%Dq)TuBNp>()vBS=ql1hz2)>cAVu;?yojb~
z&oQ-8{)=ys^YAQB)jPPO#o>A$>%>gsqaXX56IYy3!8268-(H;@<@Gy<RPI?%{<Ipo
zjV+`)1sM4hRwF+wjUf`gB`+ug*B3}SKmxCOX}}wXF#?*(4G#)TnJxCY_JNYR1o444
z`~~lIWx3gWV9=ypU{?UGEP!aRtAySP)#?hRE(wPkcRYHR?&(?>zY4OL)RbRWR8bl>
zP_dTS9U@PAao2I4wKJ;Vo@uU@t#lTGr=?WA^4WgF#8phP<jrAM>tzA`0e9o4IxlWU
zajAR_sA!;2+3TQuDd;4Jit96>VeDgqZH%a!mNWv4fMdg<Hl1=C&znK?{Byns>7ki{
zO2Z<uO2&cE#^blSjZ|UW8w&8e!S%x4<D*u)y~B6oG}@KsbiNjhBmJ^bC3%i-tGd?&
zNDC49BEuFFJV#}1fs*BW?;A;9QlP&gWJLmtPkGJVn^tkEhn-P_nW)+1x{GYJg||bv
zBRtcQ&f4Hguh<9*!+7O2=&=keEC3;^(9YvWSy=GlSo-cQjABI}c|m|KGgA@TQoE7@
zIQE@%A#!3&eT<YcRa!1~Y$i{5WVX7E;tY+2QeJ;xvKE_p5e+gxU-vFXCBnfAHNVt*
zPNUjqv0~<Dl_NJ>YhCr}_;qa0Q;x1jta!;r@jJ*C9~&D>rG(_0TDvn<iKK*(0>AWE
z{1XPJuh9d?=}wDXF8!jae7jXxPL@W7v^=UXu=Q)xNSdjuMa{rNE8m+`e~VClmks^a
zK?D5y?-&b9t^>CGhZyXapaS>|8;1Q-be}4V{|d^m@fof}=@+3^at?9KJ~MRe`GzTa
z#_3rl8RL(SRE@nPiLa8hIREI#Q%&-*eL5`+`9Bx#0LK0!<p8WvY#N{itN#B71CQ<0
z?}W(jhyF|94lcg@59~dV7ye(iIoP+0Y*H6I^K3VEMJ)7pg5poj7U{1lEo{!_S9KM@
zw*M-K!sdH$BZ~(Fyx~&+UNQBL2Yy#f0XF`A5>O8ODG2*VZ56I!3>!t`f^A$`)c^Dy
zAc^$vcKS$Qw(~v4N1j3Z6q=%KMNCaTL%Y+2&4!J-Yr9>{8FkQ`>2%wzb!_?hPzE!!
z(f|Mtb6#PuEXqNo?lce$B!-^oe+D2T*bv#cohtAiU}fks8{yD3@+RF)|H>j};G&&O
zKrSfoExA0<0J5LWCw2#BsAN1@OWS`0#_}~}`W9od4WlE~-@BK4CtA|m)~rnScY-RE
zmW=CN>=k`$pig%gzZz^H+n2>Y7cr;x6DBy`m6Azk?DVxyNfAueCqNdDK-~xLT3;{g
zxtggpD%6vEsgu-Y`uvCbORM3IX8ZPvK=S4x(zbeWq}Ih&@pbq458n%zAPEzmq)3Lo
zL(gz>Msu^t8@iP+Im}e|9kU0zzy(ZJ)4&}jm5_bc1TzaLa<@F;dA^CTAk6R%MplG;
zr0jcr;imm1-@7YZY5}PX*^ak)y-{V~ZK~)ItK?iKHsJv!+1PuD%Gwfa4jU#$du!}k
z%PToNOt=H*BD$X{JsXnsHuu`P(m;#Q@kHhj2lf>DPamqTMTCBQZeLkaCv)yeX|RUT
zPOVJm$$tHp$TO-BRq0BSW6cD8-?B1=_*E7&FE<+Hdmz`Ih}<iHPV-JB+{1(!Z5Fsq
z7dLF(x$}aM_WBu>C%-n}e_D$FFINq2ef=L-8kqOxvKV(d<H5~-y9bH{KF!`6as44&
znA!s~&)(&4S%J1(7>W9qjp^##{*vO58L4#A&I3K2zNS8u+bpH*@vRU2SDx0rQ{s29
zw)o)8n$U(NZ?KEG&g4wO5^eT<U&WZ(gCm~m9Y+0olZv6wbXK43yvhSA?sK8jTvlyk
zu_<>{F5Y8);$e~Uxezm>{{-=Bz$890)aWts)9o5gNc>29MgPszNl)UxwM`{bc!y@-
zOt$R<x-(8qs!wE9t}4FPE0>E#X)1XKPY(x=boS98Vx|uyxiF91rAVD(cW3XiPJJ)!
zWDwG4Y)L}BJxyC-_vaoSaUJ>2Vb;X`2oEIKC>Th|-0$g>e%ToOBN};abE7#+Z;$`I
z2%38mL6jaGMFY0zWl8c&lSMWpy>_31N!t3`9#%V}7jX^~m@lUfS6a;BdV05;q^ZZp
zKe|tM%Lt`&hbH!PRyLZH3BL|v?7YzwRe5DR;I2XCS;p9P?MZu$<-DuUjXN6ovwa{8
zDLdb?bgE9aY{*O-Cij~yQ@`Uk$c-=E=T%4Sz3Y^e;_YaVOi=WzwqW#Mn{>hqzp7AJ
z2Skb9Ptg8uKZu?Ox*qv1%5<?hIc(p)Z1>Yc<%M)u>(iYd{#O)M`|ow^_{&`yuTorX
zdZBzhx4OU9)w;AlBJiZ8-z@!`hjsd5Tc@a}&cR7MYm=UP8lt${Lt)|gAvRx%bwLVD
z=83M488(2sClJsZci)GZ1F0Sd-Dg*HUX*B!2i&=MTb;YRWdj_02}bL_s-ml2t+kyK
z;f#pAL}ln9^s=--fzyzSyJ(`0Yt&u3w9?<}t-q}34b^0^Dur=p21?RC$3SyoUyUEJ
zW_u02ycH^tm$_$<K@p1Zg#)y?^TA*S#-U_3*1)?+VC!vXLH!!bs7<|#;YEaJ{qh?T
z^}(vmZf>bxg{+AAo82~gVPayVhit8NsTXf*e(Durj^%*$7MnsnE|!5SCswLb;~&V0
zs3;5M!}r`chEj~Ac?oIU9)~JUNT|H1;vEz(4uMY`ckp1w*Hzn}+2qrXcWT<m6zRRJ
z9aTzVF&Sx8;~lEAJ>yhL1|$o7^34)hoSw*mtx3N+p4oGdky+6$4C*_pXu|oVe<|g>
zRb@c6Oy@HWIs`njO;PN2%bJqdW3G<6t_EIa^Yot!`6WEun}sJJTfu~>-<wEVY=+c&
z;*)f%q3XK({$XRyKF`IZw;Zrh;z`N6wfDv*&XeOeKG~_hmb|a|qKV<L>5ru6F4JFV
z{KYR)uWT@4Duz45loC)i@vTvIwb1u8jFC?o#WZK=1)C7ga-W@j-8rhury0&V#0c9^
zl<3kTl2errMTFXUI`}1Em4E<4QuZU;E58FbcQ`xkx95B<wcOcVsZc5>Q|59-F^6sp
zW`;c6XpT3B#nn#@>Mi@1;oZwWrhNDHU8wj~{XK;2-d1+JgGkD_kw1rCf0%MBr(PrS
zY*p%vt`+JH!%UzofA*jWnJ+!kn9q2Wk^9rk;TZXDHHZJ9*~LAi8MNyB1|8UG@`YqT
zd>E;d#~`AD@kz86;Sb8ufI|BhEw$2%s8nuP%yij9=|@T*T<&&l5yx%G;Bx%fHH+`d
zO_yHrF0}TH#e#70%jL37^v?JfT`P+0iiUE+QQ+@v7NZyPyP5?~oHXM-*%~G6Id`<Q
zTA+)&4qohzGVH^9WQ7d@IwWV1AzbMid+Cy~UJnf-_gafgs`)+qfyD0PcksT;TQX<+
z`H+p4r_>eJ54`<})*P}8K&h~H5Yr(vYvT8LOo^+mK{^)8Hj#DpsVzRnuO-u7IVr@7
znPmCIqrB3WL9b!j`PGXChc;S+cLOOM{2k*o>Q3C8;QjcT?3B=}Kq-#-W&!eR0?XG@
zpoWI~NgekA&BX`mcb-+cl8_;Pa-{1wVOah$q2rvQ(}>cb>ip`a6Km{eEe~Og8<%!W
zI7;979N1#jY`$tzLi%OZ8D!qskE;t@`7)zE)$k6zSg{%nA}@!ong_j=EQf0^FSDny
zNa?ujcQx#LF*EZhh^ZtUF_7~<dIdkU>yL7uJPduM=9vM{nPy+JD+N-u2xtbJ@=-cQ
zBKT~-_Nu^N_Q<`iP)Kr;IcvzUfOHqyv_kDu9bT%E;*CQ<`54yY&NvB=x)UQLG;Jj6
z)hiO=-xsUO)kDwfQTIVLnd{H=3l8XUdK9s14!&~2<R$9!8Qe4O8V`*?OEQsb@2;Nt
zFzc*awy801m%lM|twr{;KXDcBa`hOilmA<~A@}Fd7&%1Nx9(UJp9L|H30(c?L{INv
zjj-d<=Eo#ZFvl#Qt#%$KT<SZGMF5P5AR7~?UZM1OT)cP_R<~rTcrvJ$wZJP}U<BDG
zBWx>(S=6nK?XC<^TT=b7$<4SdFE{E=KP$-dZ6=viW@Ae-=4#e`tM3hOT`=z<d;H~k
ztzpH_9JRZuXo^2_A6C;?(ec_<$oMVUtDix3)*$*5roYX65c=-V&jV92+%tqlezmi`
zPMyCV19X;K34i!kkc2g??;3}=#Qx0LN<;THc2wp@ZjCe2K-KH|n`<9yOFOsq`RHcd
zIu&_&(Ld1Rg{sVl+$H56Oo<#~hT$n5e4aXe)BWEHXIne!o*7(iPEGj!eEm+=8@Zy0
z9c04p%hcBDSpOW<Rz^f4%Ub-V={$sYIc&DhF~S`!m~gEagU~xpA(7rno8w<`7?hnx
z>6A+5_}h9yCvz}?8N%b9YVM8dHYq9-TUr~;SQ-v{#$M08PDk<kfE2*eq`zS@7NGn(
zTyBl*b~fyv9SL|u^7q>a{v-Ouwfmb#{CAZ7AHVY-ko}66r}qguz4wHc+CV^4ap-p#
zkU<=B*Z;@0>yA93)!3?5ERBU@y|ChE9H#$WJ^NS1>u)#@^z5`0_Scti0oq7vgVJGt
zPnfQc(cbaOMXcr6un8cb9l}lm8;|uS&#a0L*wFwn5@0(v0tLQ}yENf<bGRS;MK1pd
z6|t@0SUBweA;bHk{S3OZy*L1r(T?z^a&zg;`nv8ezaah=amw!Dc=L0J`I_VTaKl`$
z&ANrC0jzA*2VmeR>E2jV$_dua!OCBWVSo8oT&E1_rUhG=V>U8x48$#`v!lkXUakNl
zRJBvR_NfM(2yaEvO6Si0xOG%0rQ(m~mRolz*aPPFCf~~~_wX6rwJ>dvHFd|J2BJmk
z1byyr&4bZj4snWKfFhzkS$~cix9Oh890CRW4owqHQM6z`sBU<`&sMJ!pCSw1gQ^|F
zbJu|+^6y|XkJ^e=%>6!a<4qD)iEIWP*?aQ|t5@Er>tJ)k7RGv|R*B}v+6O#J_n$|-
zq-3NysRIPWbkB?@9h;65zm{ReoBM?rB#Hzvazk&%L$6+Uz8>vTcSD6z`r=H}3(>->
z>x=4xsT#L(XOgvXQS@o7ji(G;d{hvRRa}Q>c4l)KxK|${?pRwHEgEcl&xcSGT{A{O
z6H*^X+3KV+odit1R&O7uIy5&nU@bJWYOq@Bfn4Wq_q<E9GNcwARhTc!u5|ylF*von
zZ{VBy)xxkMP2pd&k6k`N{IrOQl<ieLhWmyz`C)|(E~`feTavZ2Bb$1(5}N!Hv~c5l
z7@r~m69{+faU!^KM6{%F?g=ES^->GXumdeG{kLU64opC;db*-;Z939}>-vXiW~lf>
zQvfwJ_e&aWE+8dotXo92>0}J8FtZxDVj0taWkNmP-ie(@Nmftw9CoGty3wNm0ue}^
zHfeQ5$x;ka+%Qmi4sFUq%$iut-fJkx0aVe)#%+>*QH6U&@0xx0Hq#H!-i~p`sXu<V
z;PVa+6c0^{6rc?-o=Ha<Dxs)y9?fk!gHGp@01O$r^lR%eD#@tx8MKSh?vsii25w>O
z4L#c{58J6i-*G-M&Gogw>IQ&23l5)SD}U{5fznEI9&vzcE%;F$g$Ax9F!&%)C494W
zGc9zo6sSt=aqc_l;|4DDcq2Il5FRc4vK`EUhlMSh0EtV8%SH^x!;|FK6bHK-4|;zB
zSNgdY%x*M-L*FZO8zY_Pc_yQc9k_F3sd`rm3-U<|3$hQ*(IZHp&`po_`goSEsnUMk
zY~;R8)1<W<uvoCB)jl_#WCLW%)BLcQ{<^&9ENjdD!~tA}@N&fSgSNGfg*~h`qQ{w1
z=`b<x<mLW@5AZbLLJj)KLcPF;(|YsiB^F+fD*$OJu)%??E%w|vhId>cy)Cmdoa?zm
zi@-qa{Q&6t7&toWwf%&Tf_wY7L4>v=mgSDuDH@ek(top;i&;ZWM6=mgBaOV{1d)}J
zeq-zw8wTie*Kij4JV(6|C_#eXk}kpvpBp>4GFL<kel$Lb_XW;C>~Ah&-3m|+U{KUj
z@D4^J+L>;l2wZ#den<E?;Ap3SLdPxc5kOS5A9Rf`w|Cdyp8>An*gXhLJx6gk_G)G9
zbvobK1Y<gr5%sobCBUk>Wq~Pv*HkqV?`<_As+Rgl%kM$zQ&(*nAiEhZ@ojW(2atJY
zV(VYe3V`Fl&hW2Yl3Wn|N+>|(h3<>)FSX@k4mRGJe8O%JtUJbe)-z5-Kn8qlv<Z6c
z>psazwZs}gjh9xe*^$m-wd3-Hdj$s{+6D`3Ozk?xfVlpBNvcPHpI}i;vnHX=%6T~|
z(p17*aRXOc)mSXfficDYz6a$tfNB=zAyO|M--fOBQ%qQlR<NjoPXr88y)h((Pvl|p
zA}3q{4y@_|G0elrm#gN@eN;lgC<C5P>SjN1FUu#SNP~|eZe5*fG=QbwpmD$L#bANY
z+UCdDSZv*Hbr}D(q5c4aOF9`?zOD50xF}cQZ58Oy3ql6D5}+dTjRTUqfCf@%s|79x
zT%sfQN>w^gYhBV-+Is0DP-&?0#5kA|ZMe+khD{UxH6n$yq@+fs5<>;A-sqF&ap^QU
zNw1SyNDCc%13LKuECX14;c)D#+zUYb=^|=nOlNTQf?7wh5k!JzoE;4)Y6V#O3tAmE
zK8!ziRzd_c!2nhUAgX8G-YYVgSk=75-edusQrMO81YTt|$W76vu+gKrb>*(Hk;G0B
zdt<U>Yq6%Iu+98?gBH_11=u2)lUGw-)+o7C4M24T02h!*hQl}YPR;;O_3v$gFT#Ml
zZ^pyff&>u?hIsgkNsDqz{cB6L1`r8Vne*Tnr*Iw}1LGPw;3|@jrh$7e)Mv0*0w=fv
z3I>v&8Pn2aQ#7IWvnM_&zcEy@diCOiV>SHKNO#emlX0v2Gvk;`#{94VddoT`W9;=J
zDnT2^w{#w4?Nl-ba3}fbn4;+#fO}Pq7bO7==;MH;`xsbP!3#xnqsl)9*a?t{J|E1x
z1gnX?R8`=(mS*f+g1*N6EFcr;u;Y_L`<5K7wAKP&r42JEWo@n`aJ5F5S|;XX%y%cF
zuIs9WuL<ovCPEJUO#EG;7E+`0lvf?>uRjk9dSV%SP23XxzFkG}%yqHnr7`!ed_K2o
zIjE}K{_qSjK5a0|;4>aiqltHmmh3Edgerf??eaD8Bhg3J{x$LF8AWvGM629lYO%#g
z$xOqj4?+yq;8|AeKiglv!u%>xIBnQvAbZ7Hs=3R{TMr`x2X#CsvGMS>cFW&=i63li
zX{^N;Y)lRV6(T@Rpu0sWPtWUr1V3zRgupKQ3p-Z4E(uC@<uvH#P<^oQ6TD%xu>hH{
zk&Avy8WJ&jWb-w<pb`J`Mjt*#6RF9*5##zk@?~1UyD2q`!4Uq9F9EfYaWSFCQJxSw
zNSuL|<%p>zP_X%(7QV*1iD#PGsG2|o4mRe?CIlB@-;l$6Ac=Gd2N4?Tx||iD!sdcl
z15hD7gb4ewLKaV4B2j)j8H^03#m5bI6oQk%Ovz!`CgXs|5ZLD{aWCw*e*by;1D^75
zUF50*dR<Dvv$yin-@objIcQ|1h%L-X1#2Gg!VQS!2(~#?9zG&1f2b#jk#nh>9@hoH
z{$_r-qr$zQk-UwuuotJs0G42TMvwcDKRdR$0Q<YAjSzveV`+ds{KH;gEPx*X8WMs#
z;wFI<n1NqC#XgAZA@IS!H|liSu7TFSeM7=A2--=+a9%80I<vEb-T#qWr}r}5Nn?P2
z)8P-a(48<LDy1ney=lEEZ@UGIN1=%n%KB@El48cPWt+t4d3APojnWR6Iy&_Cay_>-
z*litW%6LXQ+3$^Btp&zyB({Bm$iv~)Tm~iYB}S|fz-XP$ku^D_Uj{Wi&uW-+K3r+%
z?F)ufUl&rZ7+xLzfCdM{bTr;Y0}DF1aNV~zWz!!);0e|bL%Q5vB{e9-e7AN&Jl<zX
z%ZG5zM2rJLUQ}aA!$K5IZLVIb=`r3#>`X=5ST?wfhs{@rElYZT%^sz<nBCS&pPV_I
zKt?WXdLNC?(~{Fm>?zb7pmm&Q8-03%BKE`gi-Z)9O1<00mxV4PoPBQNR*9CSGYKr{
zej~<Wv{(pPy;0VENat`fSc0VMJcMLI)wpni_*hqZDbb1ow!66=qvuk;tWMBgzIKSj
zPA5{70*1GCZBxGD=lU|el;5|Aw{2MsybR11$8PBpX#p!=+ulrq<4a8B;l<Lc6=GeZ
z!&1jOv`mY__3oPQ%>w;ew@2DyF*A3k>gOBF`ldZ+-&-`FPPPP3FrD57rAB=JtZEuj
zqLjSk5DT(fb}3%>4%p6c%W8|ql-1i=Fj=)d$PK7^{XIz?g5Svak`)~PR^07gDS+O?
zW}RoIR>(W3{j=D2mw&7dg)eV5d?1Asa#SU6HZ}1H{Ukvm;)1G?!gaA7DN=U2?^zFT
z$i?OcOvpT%s3@D3PP>}IaM+nBO~MC1UPctPRW7GneCKr3QgAR8_ve~8t1(29`7xH0
zBSpP%L7oijF02_LZnYb7O8)N?kE;B`wM$O?42K`}zDUs8ku6zfz7secq&inhraC^G
zZxOBVq4H57rXsy{<ye74{DvcAKW~2%<kzHZP{2XP!&`C8u7Za8np8Q0uT9R5;e$m#
z&|D-C#Mi^4xI*|{nn(G~A)aiMl^hi0Ih?-tbbFYhXMo-&2eljP;Y46dkQA&w0dm1#
zTv77RID1heJu~%D#4abR$uKR(Yq66zc>+I{;M03yyjbfo^`=!jGFU{gq;&c)Ny^nK
zY2FH=b<j7dyN~p(A9z2D#}oxX&SAZdIvHH|C$dR+=D>nVM7Z(r1<evM2(lGSq9cO*
zCh=OEqj*!%QLQF*6hBV&{_NxJlzR#0kA*y$q>7~%QVtepLw$AmgBczYS@Mw+4vEYV
zR9AFwt#d|xuv@Dr3*cx}w$Iv|!K@s*Gryixp;Rh(fQ*e!G5<cf+mN0B-Imgk$VmZQ
z1b_DMt(B2OilbN6cPuIF$q8O4(_K`1_NpyGI9;iU+WV%Ja5I@DQNctuAqgCuO_<xP
zB`fH>{8~SXWVawg7*EdpGTCP?AF^|0(PwzbFSJQtp!K(m@a7la_;ew7Z;UFjEYg2~
zDzm~{X<VIOWLZ=}F*Do4w%;X%Ge}ygWLONsH>~2V<5(m?Omk4zac55uzeK9prw2yc
zW!<96n=i7I3`#6y^Db~ZyJ&rvznN4Slp)e^IJv<A<AX0zBhEvDzt-$9NbZ%g+EZ9A
zN*}A8kKd;r>Fv;Ba<AaSxw^_$&OFXbElC6d`qwTB5Xo?aQ3>4vKcUUf_vWHbG;Q?S
z3c?eBme1XqF$s!UAUGdfC;nV|HhwP^Gpxxf`ZCVmLD7>1bJpt!@jlvxB$ir;yTB|?
zOUHMBcTT)0{qqh!dUIA<{ecroMLDsNwpRkU7TFm2n#GF3a@u=2HT?aFEqLo^yX!un
z@_Y#L`nu2J>qnb0->Y<F>~=!ipRjWy^pTnoJzU;p51rh#M-RzQ-dMe-*(@?CO?-=6
zHaR-k-?&-J8|TLPf-9*!AgxVf)uqog@M4TCl_|V>gC#7^=an>*^%x5!o+qb*g20&>
zut#PR^h+-cgqLz@I<xzxRCNrci$Gw5$RY7>SI#Aa1o9ibF)2;_mfZZ^qO0<wA+iKx
zxFbn}7?Jvzt%>5OZ`>s~%T&qmiDArA_jNVFcGdH-_V#43gRKT4#26d(#G|uR3YhEM
zLB#N@-;W>@gt#*aIBXG$Ac5u^_pj=Z>Ka;*FhTIkS^7APrHDR)o4YtP>vb8_$;$Jn
z(LoOl67jd>naD#7^mr{J#lP(`=4WxKE4_a}nZEmu4zBj>txWZ575JP8&Pj7wI#<5g
zRK^!*)-ZRFgB{Af0$S69vGs61yvIHO?xOJrd~dz2uCtHetzJ$S!qbBvYfE&Ml`zkj
z3*mIsZ`X+-Z%Y`P6F>q`8qw@NzX5J#F(GoZLAHzPN$yg7AA4Gf%0h{Uh$l5Yc!;Ez
zqG>{H%$hHu2B|ncvvgdL!x!kldlT{c^W}~mSp|_S_My$h<*q5n8r4r22;4`ZKcR7h
zI73rRcexAa!HQ|>8wgmu`}l&qMs&Khs~?cSR4qfh+#is^dStSTMPPpLh>#o)h<5<L
z{s1I2QTCoywr;Z)t6<qDx7`@~hqZWdxcwR1JfUsLT}I}G|2T;6%ZK?Sr>;j3%K|#M
z!3*o4A{Iz+sh28cP}2TW>tM;bsH^MbmPlgAsz<XO#^W;jwPF=dW@^M!|6Mw~$c~vu
zAVkQ~Ro*yj(Qw)9X0itapDPbD^-4J&_D~grz<X`{JzLeOA)kMH;M@8P9|Q;<m1&9C
zUgM#cAp7#o?drjnGUsZmiC{xzgkL*ICvkM0;IfP`o4QJDoL!!p+%S7|Kfd3PEU3cT
z#xg2fUf?W2ZG{rp@)Qa}Z@uHnjB1b}mgYJfEA9iM@uVgQ4i6`m^9YusD>ZwMA3gs%
zlVm-Kit$ip<`aDE&N5Uyi(8lw5DwyCk;|IC%Rd0ZA83Qcc-RO2XS@VJ!vFrrZ`AQO
zNQFaEIMjv1X1^iTU$}Nt-eI$IYx5=B;=jW<V*pV7L7>2UEEG1-qELYAriphBbDBF8
zQ~aYZD#S2TD&t-7-_artll?aAU%@9goH8UAwJU95Xp#hg%)cYLLP94KQUEmKURZ$q
z3tj$84*)cYz<2K-cF-Na@c%0ig~k#f0cUw~GGa!36L|?GOv`g*hVAGdqrW!;#my*q
zQ|KwwJq6*|k8yY#i;{i~Ar^?sItHPzNJ|oCTA3Qy;4l=lAL<jh-6JN1E?xZklBA+x
z)^*$fhJ}F>JN+W@=I)wXQ)?q~-+Y)q@2D#PI5qb#h>SC#gAJbMv6dgV6VS0yVTi@I
zs6r}%;+h=7whFz%;-*GFij)A$rJkIC9{KqYeCCrS^rpz(v|akWQ9v0SrTFdD{1=Cz
zuDxib!<G5R$x>8@DB=oARdc&^zwMwj2JA%pa1kJLHZk=-593Xe>QR-WBrtaDc{Yzl
zG-VsWsW1{0<vX2|KvHpnTX?`2L}^*}28Vq9Hs9};1ADKeuep|;R&YjOa#qx)E8_+v
zc$h`E3AOURjje72EGWA_Wkm&31?#0s6vDJmhxTM0s9CL>rh>^uGMn_IHo^OyElPJ0
z^LrysU8CAAcW&Pt-`rSqQr-pt!u0T-oze#{Wh>~1>HW*^?f@;|0;J$dhUTyn4<zDV
z%g-&H=7Xb#eR^%->WnG1tn=_lf9#QZgg!*IZMTJoPYTz;{FI7vbDs&{HGE+m0CzEW
zsM1T4E}bGa0=0Xu32Ztb+rx$V?|N@{wtHqsO)QJp*gsXwUC0$VsJ`qwlOm%tX`ftS
zHG!XEEr-brSj!HG!fX#_>ujisCoC{ZWxOR2z+{UUW9~-;$>w*b+-cYJZf07^DZzOP
zdwc|V;L{AAfbNHB0RGUJViB{RKj;?zh<po~Mh=GQ^BE+==AAn?111}7FsGwjM1)Yp
za|&WO$FLZ5*}zB?F5|y8OzcN3L<Aqxt&u6WO9&hfXb<I;%qApoct8^3z^tr|#t+#}
zs<lTxvyS33j3w|3zP6U+HfCjWKCWs^k98ohYKZt^;uGk1z0?37YcpkWe~!4f{>_{l
zUlgsaS=N{RS*WrGCH*h9F>2@q(|yhjIMh0!91A?zhN=cIE`J@$O~~(#NH{*ex6n=#
zK>)|a2m>IP>)Q$ww^da7ETtKm0QU1$YqX=PQgD}^h1YF205)k^crCt}$KvKGbuJXS
z0J~tHv|WJPpo86c1`B_c((d9qdK|3WOzTJ&QY^L9zrE+r+s}u-;oH0h?FNf;lu}13
z09ZuZ0y?yv^O?_i28buz**D~OcgZ6$6{Ugw_}unbHwNrMdPp4}*z&A&tEv*dR>oQB
z@Q};5i2bq$z!Mj16F~xa%;%DG33ELlJF#=;A)#Q9_bNyyHdm?H3{POu;Nkr{K^F(^
z9=1wGEl5DG#7h%P-)GNnDUCe7A%_ohAQQdVDS4>IoRV;XmiULECDARzS2=+X!W+ew
zE;#voA9TZI72Xp;WayQrP`!0=w;1Op*e$1jZbx^^;_J__%UIjK?IJ%1N>sh2iMC<C
zCFj<!+4fRNvg84KbxOtlxA~Z53$rKINi=644vMV3OLx%v(V|@<<ddT-faEppLTYH|
z$Yb!0Ocb__YEp+SGCxqaQ3PRwXuu08umnLrPo>{TfMn%4LQON{@YurnvMq`I)WoYs
zSP(A3ac8(*_g4rjM=yky5yNv<>q>=mSfkofH7bdkL+^nSsT;Tov|^>rSr1n0zVjmw
zx+0|{gCtTrm?wW~GtNmq?^)r64tQqq<T{^=Ts&%*;V7G2+hG(0ye0DM-1L6pScteM
zISh~I785aLXcjRcDA!@lg{%f|85=;_Xgwr@d1L=mK^oDn$dQ-DJ>-JtwEMEDsM+Xr
zLstUV1gSyX*cMxr8D3vo<A%n?PO?CTG^_)I;x<;@zIMj&#C4c_bZ9JE+mb1LMNO^K
z{AMo*GSN+OA@`vv5$xbX3rv^gq1Nr0WfZ}pTh!p0^(YXS6Q%~KWGgs>WOE%am;U;c
zo$q}NetZKyc<?cP_roNYaCTo8C$~Ls@-Qq4meGq{u_c48Uf1%+M~tQQ+X)34O(PhB
zB-U04bN<?veOA2;N|}m-E}C&VADyoR#mS_S_sKfdeRfe*1TYz#(V@<`UnA08Rd(`*
zlpMZO7FBLgzf)#)1+Bx7Lzml#Ru|4KKmfN8PLFy_=~R1#W<Y1@9Vj|$7Kk8@e$Idd
zoZ*<x-dR^47Tsp-p#T5~r(h@vfxs74BZ+Bx4>dV_<l@e<2}r<0`VIgC&*|u}YpOhB
zICp=>o)-v?LQ_Clt539>x$@hGd!5z@r3=^aB>~%X8J>oXyjiIKBDpN&pC+_HW1j7b
z+c?pcj1bGn-T6dja*eA^6|8VJuP?h*%}n_Xz$)BTp>p;4L=i4b-5aTR1_nYV;(=Yd
zsu>+W)Mg`0EvKoQyIQnLX@p;Oyay*fC0#Fls4rld?FZXUCU6R36+m7c726=muY*;E
z2b)}s*?TKB>DcJl7<stLzCnA#=DCGI68?DQ<Ea6{XgtANo%32*<Q)a=sX-ERNwN$9
zM+@1U&a%pBJ1VpJv_G>HCPM7`aBvWWY{nPJAj>Foyt0^B(scv|Bb|w0S!ZG-d#-Eo
z!9jHo?E2P|C`2Uikr+LB3zz8H+n+;60)g!U9VQTv(wj<oxN|;cpyum*j);Xu)}4+8
zM&iM#hm`FUR8sW^UFiq&d=0m6+H%SET@zSVzB0C?=AGdhPlWhN1j93y8l|IF4>}83
zoTUr6NpRQNipTQmMM3}L%d^WG<V&uxp=Wp}HQx53sZm308#w~!JC5|5tEMF<>@Hni
z!y~)x$EQb|!!FoDc;4C|YDlKUJJonnnuLY?hnmu0U^%0h@=bPWb_ezg9?F4n__M*s
zGdSPENe9vFXiH5)k;#LilzBH7#tX8zmOE&#Wy{YcD!!DM2>01uh8u{W?*-N>6me!&
zQs-~j)&a|hQrj}p;_!ew&0;;&9uDG>GEhF-^toe!vlc2JV3pyj0TC@XSItm@xBs+8
z^-N8jjUuvKl$|54>{{iN0YVpOUWRJ)(dJVuJW-&*fkxyxkO0ucb=HGMFBkS=(*_T9
z_3I?Gh<l3z{8y~Abu2wA2#}UX7jP*pob@<Vj8L3?b+w;=Nyk`b$@j+Eg?wjk@4>zx
z^1)sWed}X}4!F?ZUdY6@{B?s!O@$n`!HvOKG0WHD_cLpIdNci!*3`}UK7FA&RM#?N
zX{{39pjhD%64@iL5uCUV{2N*L--8^|=x-V&ENB#V_wzZI7hG=cFICMmg!5jdI3s2|
z(cdxTpybb6%e)-ZAGQ+{9lcd9!fJ8IIKX?T#=a`xugl=HN&(;YcKaTZsos<)qG;Ft
z$u3qgmqzSybZPsPeV+jJbRx3DE>`$a41qxS_^lUZqAwDu67N2Z5C|~)-2I7V<A|vn
zjoYuMNC%71DG4lXj=@F84-ddL^RFrKy_9v1kD4A0P+RiPMQzmE_0hfR5@Q&rzeVo#
ziA5|b@~Yk})Pz)AFbrh$={@82XXh-zA>OFaI4S((%LK1(iR`o{hPQKEu1IdVg(q-J
z3cv+9rJ^qME>?jPo>>ZF;ER!m4!jv6;8j8vhXgkQdu2|%afd)b7L`=x3(j9`m7R8<
zdc;OWH(qVr70ZnDisG^f+V*>{-_?<_IHi2I(NQ3sy@TMHpRJ!R9<RS$VC{7QV(@|}
z+ivilj7j}c$GNrlX%~FSU?~EM3>=t>PElpqWu^9WqZdaBmY=XI-TTN=oSjJU`5>FM
z_+xeZimS!4BX<7(mhF8?2y=^}T8AbJPXH;q<~R2I$}z(9AVMGjya1wfh^6p)R}?}I
z#uxbUx<sCAjPSzZ!h7AhTr!BYTlOP2dOVWq&fvHWJbkts5f$^00kP-TtIE%jTd!B7
zR|WUW({y=<?XY8X-iNTLKNIc3tMz6)J4_HdEm#{HFyqEHn0G&ASwlb%ofv?i&R$k0
zoRi<6vex4~;DfhE6C8+f2lrl3F?*_-Fm$zoHp2|c+Aa~VHibQ8|F(*)2w^fkM%fhm
zBoO>?mRnaW>6*0;ZJA`wV(WR;e2trvo)V*Oc+W*|9~|9@$_Ih(64X(%5<X(|g=Mlh
zlRN-je?ZkSe{)x?IWa_Jfh242pfq(am#+!$lD^6zQAXQO2y3OIpQ`_kM{`w0$!_XK
z)<nAu9@l}OM{Z%((p*|vG#(;kO8tI};wSHK3f4<UxtNd5T+3HTzyTNLNS+kL<vF~(
z9ANhJK;~NtHI(93RKKi5I>Bf<b_4$v-pNJ;(Lvs^yw3IPx6$d|C4$@$%Aox+=t5~j
zmSNn?)L{}=@}#g}nLG!DCr$m1n~ZU_&A~kALJ9u&3wPE+!rTL7C@+NBb3bI#|E7!P
zY-1dvU*^py9QwI^)ff}u-O=#2b%FxkPIZntIA?Q}iaeI6zVMKC^tMm$mlFPr2xRcy
z0#grg7l8=sHIw;>OP%)=_9()^nO`nOsP@y;bvz;Z)=rA%Y(A__<YJW|a8&2=3pG>c
zAPx*(`}WnWHnj;m*I<b*_4A@d<v^k)NhrTUm55U-@d?!{lR_^^K6tEfNNgV_i;9{$
zvSP#&Ei+5asB=j$QkoW;ebZF$m{Fg>W>9=AWGZKpm<i&Lu*1OD=^*RMq}dzDl7IM`
zbFgh^+pY0}Hl9;s5KRZrp`#18h#FJS;L)ak8PI2dg>$c>lKa>rYo4xaGl+sR?gghl
z(og5Nr1)}=(q2|cRERc1<9=k70UX~_WBF|Lst_E+u`3bz9bf$Qn~4H8JYITbM<Pd6
z3ci>YE^7U*hq{`)QKO~o%J-T1%2M|_{KoK>f^Ht3UwALS%$us+{d0`o=I719nwEom
zIs+%c#NfkrZ`&w0%Fxn8BeEV*D#-vsjbI7O^U8YtL`qM7sHmI1K#FY;Y<xn|{zyyT
zc~mNhT@N@6fE#=01$+TwF8n%3up~$3@HO$P&#!X4!aIi!Z3H=QU3iU$2oh2zT8Z2>
zjp7U=yN^d2WE^y!T6is!3XgaWKZV*n@{9v87g3=d8CN9RNnceFAcD%yKk~Rm;FBk?
z^tC;#0)IcH{DNK&d$!<$t$gYpiN=5lM<{H)+o=oh$=2I*)5S~{BiA*L;LT2oPajcd
z0jFa3EI-LElpMB7#dvUCqm)A&<xZh5HG2J!KsimVPeL<YreAb3)`LOmAtC%Kaq+mu
z#?m3_l_(E3rF)-JEDz`TKmPJQKsc>C;p^UggBP`OH5~}pp9D(`nuWT96!0jdb{!sx
zM0bHG1*ZsZwwd*a-&3og{;csXB#&be??P1PHER1vyO2SuHC@h{W_Qyxj11RS+Qz|x
z?*(QakY<y?g)Zc!GHp2rA3n5)v~4iEWsE&HG>5u51VkfbBcCFWA*Q4Ey(0Zk6F(>5
z*nI|H;P7=0+mPOxm0v_$<F|0Lfk4IbJOTe1r<A9DqM5^6GT>Aa#m7~PUl`I&(%!HR
zxVCpM_xwDV8e&)b`R<Zl4RIeM^D^1k&Hih60ycP-T@MwvjqE2ZOL;OR*Yel?Y;>Pp
z0)QJto&g~~ieF5~HEWmu64Ffh0$(PBi-a%J(III^LqeH6XlX|>R1tLFdR0w2Na6(F
zT6!uYTEde1-iAm`ZI>DI-<HQ1fM=-*^s~qb^0VMHR=u2-1P&ZkS#)C2H1<4N8sgE7
z@C#owURGRSU%twInDiqC5AWs+P1S2g7P5MVz?t}Rr-=%nF>#C~a3mluNv$?-Y0z_|
zWaTauPEvx;)7*;kPR+`Q;z{K6kx%LvO=iELvD0Um`Zo`JN_M*V(<&J4&64~k8gY$I
zocRY8YB@5IY>!LQVAnV>5Mv<3$<O7;w#}2pws!OhrpT3E$#;v<^;rdw;n}O>!7-FV
za7rO~n%~Bywhw9Ed0PRfrzUhJGr&FUWg^biC0?60=!8Juq4w_nVh2a->YC~5_#(xo
zsSihKH)dD>_v+N-^rw5QM6jT5nxpmQ4wFyrDBkjeZU_+&!&IG4mbTO4p_F!(6p)6K
zugC_`Vc;tPqc~<52>n?}VTEkfnS4oFj#nOPCjC&xM@CyfEZD>bgNz~Ap8WObB|?OJ
z(qT``V>-LpA5!K|r}w24!H8>FdxqV*f>w|FX4&<p&|f4X4@v{k2m5gyDK)zd+=Oc8
zR7JN9s@fW(1!Zde(yNH%MQ#HUlJ!C=VOyazaAVF*QxY2&^fpmj{wOQBN?~(pAv{vA
z$Z<XT*neZ|PPfKYe}W9T$U<6}i`RT#nb1VmMqu^k{QQZZuTxEQRbLY-)?N6a&)MZ?
zqFzr|bt<s=u|M?h#$18@8bzVvh-a}arRNsgIkTNvSHZhShYD>_i=@%Z1QiOH5jiFg
zWqggOI;23L$@R<UNY)ki#(EchIT{D(zjPd#M%f)BZ#?#^UKhb*vH^Tx-b=~LJ9>Jv
z)5A)ZbZ2b*@Z*cx>EKBOk1kw{-1@@fJVesodwhb@(~Coovvqvou^HTQaLqi;B>hU_
zIg!(Hr}!?o=}hAFd#*MX0f(c`*RwwfwF(WM1b$e2K3^?Bd70sH7w4-7Bv~hm%v7-*
zc+CzCs`m&Wg<W%nCMlB_Om;<F3rDIsv~Hf@FY#A%XOjKQnG9KTmxp)xuHolcE8+D9
zm{{!b;F$M6Sw_#TbU!v7f9TMt)~#;C+jaN|7I*wva5K{nHPts)zlQXHGVyrk5<dhE
zDGA#H+x>DnA{hTh6+z>4UrnTY1?gvXBWaT93*Fq^{q~MA72h63s7B-?M@1B7gY5Cf
zlC}S}pA09R%L7IRmP2`SmiPjJ-u+YHIWZ}RR+jFmk$V3s68Lsf$OB{6M5E;ykP{Q1
zttfv6ZBUdm^$7QqJ0m=B=2gUQqz6-pCimXb;GgYvnh&1%s`L|>T%r$wd5iVE!Gs8D
zo<Ly%|IiRFYOStM=pQx~@`d=YtaAjh-l~bQ))z~q3Ykpe-m?t`ik4VY=NHgLYqIv4
zfO!glg`Y9O=U&u+O}+jNJ9hdFh$oTVq$Y}xNhBbJDcg~GSKQbY_fMgWB8o{|COsNR
zm$#AOD5yUHu7!78)M^Zp@-Rn}!%9;XUxQdCLyAF}t4Cmta+znjJNL|4$Nts+MGC%K
z2{}zyOT;g_W(3m_<f-x#OnLcHUi@6I+eIDOvmDVbe~XQs;0jIlO$Kwa&x_Jgxom3-
ziCz^tfx&bgN5gp58L^Fc#TA`qmJ_fi27M|$-Kh3|8zyde&VbPzd`PSI^$dAdTlvkM
zeXBX)=Sp`8JmpKo{nhxRU%eAq?>0e4UgpQN>@r-@0S@uxh0qJ-muWjH)?7a<bPf4?
zvrzbl<njCC$2-KdWe%#3eG6BG?x+O)>-vg1N06#&Y#2mH-*0v}v%$tEmZB3_cW_DB
zmljC9|MH&=C|-I&{+#vwy}^yH>61@8k?t0d1Ix)WY;x}Q_pG75A1r{2brLAIAM6}u
zGbrZj>8i^*S3XDM0U0{8_E6&W{mhaq%H?oT?7l+0UZoo~GfV-p*V&VC=+R0i=Ebx-
z4YkhWH<r`x#NToMNR(!MibOXm_2fKUj^xIwrlzBdU1RUKPj*$=1RHm;%2?+tV@dN#
z8G|6rd>i4*&4=6NPHL*Nx9vk^in9cKRj{5#R`x8UJ#cTM%NbHxpzkt*8vP)I1`?oM
z?tn6+#X}IAJRyfQ>;bvN*e_v|ad$qfE~?e|vISOI568PR+^zX^`V4k3u!)7g{L~2W
zy<>C+rkE6`yulqXGpc|F2lwt2RJdwX;)eT=L&j#~fKlCetn*tk0{Fd+KOXQ8+WN<N
z1DN>txxg_Hz$tg4X#eXhIX&|JxfM3V1C~=yHV*^mOp%wiEifY%7~Oxc7k`vF07(J7
z{+;&0^$XjN=Uk`4jNr<^G{n%tk<EYNfz(`4x3E|MumyPiO_8I{4d1(&YAe}uK@>n$
zI29AlUQ`G^n6uhi;3}5Y2Sh@DI|BYD!@^<y|2vu?EX(?9Zm?kXKQ!v7N`QPrE_~UR
z%}q6CxhKtB@VLh=%;Le+Sc&2hRtVBmY(Du%Gz8Eou4^4F(Sd%-{T~w&goP&kI%4_S
z0M5uF#ecgKs~B?d=x(LjKk({nBkf=@Jn|XJ>GH=Y=>3J5fMHi7#L+n1U-0w3?kx52
z@#v&LuHU5-hawH_dijxsAjD!gL&63nJ=|cnI@Ob(J_QOc1ynfQdr<&!<372wKO?t7
z-9u3bMjd!5(yq<9jwtrM$F5TaL@)~2nU|wdM9aS}=p9;at&)ofNcwLs`@5il1fq!K
z$q$o7B=?6RL98mrde*w%+1b$-P^}y)fDiov8u-Jc;^X6%du;^B-xM_%Z?{OD{qK4q
ziAh6CR(pae?@3iZs&vH$!qu;ou)6PM+PC{B<i6viBSZ)?J%F>prLzgO&;Ha=+Wj%#
z{4|GQ_F$FS-_iFdnddxK$8>NAa6Dni_@c`evp^c3M)>^{*6je2C2!?+tz>WWinhex
zhDhRxTOIu*-o;o>*Um@K>Iq(=B?xeGX{gkX_qV9oYR7V0VXKC@@wci09ao&qc!C;4
zbTU8P8BlD!1@stM-`~({*ElFBv}Jnq3k7`e_Na+|G?^D3aLcyc>ZL?BdeGDOQ~Xsv
z!p7}yF*an6vM4-mR;V=DzYGyX87{4;P&$NtG2qD@j!5=+MSv{R7UtW(p)}-Q(Gd+&
z4RlENU`i*&Ogv8aPs+pvu9eM)43@p`qE`4jX0h(m8E*yfXCmJL#nwHfQ;AfZ|LVvr
zJyxT@cpKA>V2|rili?_e#Sahxls5BN+N4>~W~debGWJr>ZPr43#?sXstYHd<xpC0k
z4y^nQyX9E%&g5lME(IS4bjQ=A;9a()_F!49#A99z7R^jb=ufvWbD#0%z5ptTC2xMO
zIOsvJ&7+rORHXvk7!mTF#_B$E3I0vX?*w9Yk|>HZe)i*KJP&m^t7GvA@!JbzvU>)y
zshTCw>42vuG*FhYjF-i;Ro=xh0t7Es#cVD|%fw@L`G~-MB+g&-GSUOwW<bS^9uaZ=
zv7sK<PNZ`k9O1ZiX&MAw-dYG>eOX`k?4x)?l~OYqOHQ*EVZ1x=FZ_InsMiIkuD14s
zK(p`G%o)-E5n^*cUfd$eh8c4Jh|LmgE>jtOuR^m$UKW6YGU|_8x=6bHH|4x_y#+T^
znTN5wQrCh^ygmQqU@4#{u}C2S{Hn25K5NnJu<P8*Ptza)<+j-?c<O*|;i5wYRusrc
zdXvG5ERp;Q0sUL_gfO>sP$fUNG?6I1ItVmW8*~?t4}l;<L@7b0RYb}inc5Nf%J<oS
zB&a!-D4ZP8d}6d_Y30x#(2aQQvmF4Z(pZGn!q0cKdB0=rvyPBCdye45Eh=aVti*v=
zTgfHil@Vm@RP-|ZF`$h)lG_rB(K@qE0T0+d3h%bQGPj>*9WC<pl|8rqt-Bp5MpawK
zFBS`v0iJ_;I%^{&lNjFahY$x0jQhMMT=#uKd^XlPXf)4ya3-eP{k84csho@(3KsRQ
z&W5<b1_V>^&}Pb)XPhWu>9UyJdw{x8JTpY}gjTtJD>J+c1E`9A-nV)mj0fgOQgtXa
zh?P+cqO<SeY+0?kGN&<nJ1A6^!=QWCZ@qMmdV>%ECgWr6791axfqO4k_3(t0@Ul>}
z-aPkMmM>eF@x2@rml!)!NDZjPw4cTKgp@Tpv{)nrG3a*!1m}SM@%&?UiTBdzdLoLs
zv$R6oWVOIQGNtWx_8g($?JT}cL)Rd>p_dhuRrsa)>dj7BytxbHmc97c^A4*A*tikK
zmqC~TdfcPVeN}T<lmsFFX3WGg(l~r4;8oZppqoo9qW298=r}T(6E(jibCZ=X1xOR_
zxZrozM`}Q$6sc4DkR+aUL<n&SEO4k~li>R(w`m5l4O&yqB3xemNYb$L<!S-h*}FcN
z<?ekxfC#Jc#|Lxd)c4`bw7)_OkU~~nvxu>02zIkI;DIv>ddhDyoZznzlS4xN)Xt0F
zG<7dZBc39KEJ%20?Ucgv^zj6!N99=)@~(zo)IXIDi5n-4y4}rukW|aE+WmSSFgz#!
zV~QB|?DrAJbD}ID0Hl?UStKT`V&?$akXDws`h=*hjnz^qAh&YMB7V%24D2H8nVEDU
zh_hFsnc|@ho2^u{mMZ;N50JBPxPAY!#Us*0>lr?B)*HTff({i~Jda~&dZvumcq`vZ
zg0AFq(i?+n9bRhiq3&IzmnN|X$RCXDRMz@7lh}!*f{5VYrzf}EDD7i3+=Bc*06fpf
zVH^&u(|TxU8QJy<B#5tW04;&|I04kbjn|1MNi96DjRr#(M5R_UwFL`4P2C1FENw|S
zOt`rQ)6qh4Gy~S)32yjRlGq!sg1~r@u_l?tS*>c0PFQJB0A7#`yzEe94qPa5Id-2w
zV@Y}<(|jVjEwUDvfYfGd!Rlw;6xf|dT|@X_l{}{*B)}7dfMk)L$tht;ClHt;uNahE
z4sR0Fr6X5B5kojnr<fjcawKSUvDBP#?tCIvv!)$n^D!Im+8pgUo$m5zncU%I#g}pF
zV_2ENR0ncb1}2#I3DAeh9A{C*>ksCM6mha;$AAP&cI@B=?1jv~$9^~w4>6N?`BG54
zaQkqlGPu^=H7$VrXLjN46563;WNZ3@G*Q&;1}Pl}T^+Qlm0MxsBlp|@ceWMvLdA6r
z913}l|Cc~o{oP-QP2Af0r82<^rLp7okHqXBA^0hr!G=Ho!y~^WDA)!2r%{|c1QbEQ
zXTM?SpGJZFLZdI>OcrKo6uNrtPJjOtoB=xT-$?Nk3<Egs^w*h~EK}tj-mI$C4*oqK
z1fZK2XR#FPN)Z;{VPF3)NB^aF|6}K0AoWya{&&R<4k-Q_s{a|Y0#V^#B6}P%{>PuV
z4fNM9|F^cVUgE!e`4?hcD_seV*$dsQMYOh<#hf~Z(+B|IWU<n%(+J855ba|D=V`Et
z6$t&A5Crh}UmE_uJ=owqglpq>MSxS$KgJK>R*kWfid#(o`9WM{eEMUNf*V!c!hw0S
zb508UW=%OjZsYU=K49$>RN@)~-U1e$+VhXMSQXdlFW{!{_gicY0}#yslLB~W)RNAr
z?FQHyieI+k`c-r~7ud1*{TAD$-#`DixBq4dw*KIMZOLCo0i*MuZ2G@u9(LmX*X99B
z>i1|fnQm0}Dum9mH2W+cp4^@)+nb~)nfe*iuG6bA{&=DF#LVoN%EI|)6mQ^_f<2+*
z!O<{1b4=R{rIQsY^^rz$*T#}Wv6D@vV>DV4xZ5nfe!FUZEJkVKO1kU%f?33_KRc$W
zSb#EMUzEZ__}**>gAnT$T32DyqwTKiWRq3jjrE0cwotU9+iKY;#?|eo<Moliv5jLf
zb}KaH)Y9$H*`#$juk-72p2^GHiVtHq&|f4i5`Mz<t6UZLJ3{Ta(Myp}{1YDR<a}b+
z+7n_TLdU!8^wacf@0m-H?M>J8-k879**K~Gv9YU^0LFgML#HEkb;_(mrw`~|Uz?P~
z^aJ5~E2#Bsy>^1c2HI5{99ivOuWR-7%_N({x;1RIG2F%b;irwdM+eqAaFVHZ#pcV?
z2~P$!#!2dP;At)E_G2AX8mrMl_3S!mu>8)Zj#B(qK-OlR2d1#;g3!aqzI*A@I^%)Q
z5~x}l3(Jz<2hTh<-eccuS-17b+DuNLe$dBigVyc0{2B{%$&=WZ?`$v73CeCu4A93C
zUUX@;UF=43A_gMq7Q4Y5bCnz7RHQ=+{{JxP|9wpofk(2^YWKH|E(6=>be}=<PzdKn
z-{cc`?~Qx9eT$RwE+OdsdkrGdXALl$MWfMr{d)~H_a-?}Wg?Wyz8gpLSL(sKodG@4
z4ZStp9kqGU&NFX@2WFk}Mr8c&<4vrnHPm<xdZ6!~yv2DPV9!R~(3x5?5xdokX+F2V
zWrqrQz?5}p$8DXO@Ze-c@}w+tT8H$@{)YxnHeH70;K}-TGbL&F(e?Dp#g4F16pBqK
z|9#WwtwN7lhd<5Bb>D%W`VtTlGQ8NSvz0n|R4n`I-1LD?nd@av`KJ^x1!!W+OX$FN
zWe~DfHqq!{*XmKKS;FpZ%W2VOABem8o>yV<sSmPbjX62tmRj>(+kA3=c5%GX4SM0N
zNwn&B6aD==NT~@|o#uxw)9A_-y{x%D?YUBB%Z+^F{)Xw2@D18G3$teIck~o$wd!VJ
zP3dDGS<0Jzt)K3KoTSc;#-w%_(Z*mF$~N5C^a`R&jT>Bxo0O{V9;@8;o@h{5-6`!{
z9yPIaMgi9VtveCHRu8^=7DEGrQ06En$q9M*4OSAwet%&?pb10sM9eT65w>|fC7iT#
zUwB%w`1qiGS-zO!s6F~3t-zW-);$9b2w{~k7sHHFFFUjFj9#jl#C7MfQ}1eFU;)Ov
z_-l1ovd6J-QG#Z-NY9#0z&GZ~jiJ|p<3#XAi2hQMrowEGrSPG<u`@b$F3l7g_8m(1
z#mx@9M1^uY%LP~Sbv0^k^FaAGY4&02!*+fqhXrISsnsXOo!DSJ`mBWYOuh@i+pB!*
zLUM8@UAhjpI_Nhj3E`j!W>YireV%1sGepNQ^BZ=;ZKvg&b-Aq$!GgPSG%0tJk1}F&
zcTVmQZ!N6NzN_d#LNg?%A-^2E>^iD=Jx6A4>Gh$`MzTPrqLQ`?oLw`65ARBo%!%PN
za_!jA{pb^yEUF{R?`{<>Tv#hAQZZ1R;M-W*42<0jABG<3NZb=M7q>SgW{e(2C$-%D
z(0n*8yKZza7Uq_cb|ds8Hi#uDHP*lAZogBfjF$<K_;Km+Tc!V}vhRRuYI(Omp(-k<
z6h)fS5fG5xxr#I?f`Xw*6hY}tP<qg-AP6WOLX%!1y^9c3KmkE|FN%N=ks5mW&k5k~
z-v4`Vt@m6@mpMr$Gv~~FXTH6^z0b?-;|n+54RJ6+yG1L}SDxr)c@*9`%0zJ*NlL=@
zIWOA$Em@CZiBxZG_Wji;?3ZN|27=myH#;A$E|34-kYZDB8tx!@F}-Wv4=k2>!FGPU
zs%6vlY7kpm2mTvCWe(O~+Rq4&uJrSlJ|0`%O8fQx?x;GwWP`7oeT8IcK+W(Cr{ehc
zb*je7u>%OaAN{q}bX$S;QOYo;HKR9#l&cAo1q`lAAV=7)BI|KWj8?8PYXfFGHi^DN
z;aB2(V<C?Xjt`{tU)NM-g{HH{!;1JGJrwmX4W}eQ=36_lvf)o6-#Xp*VUcstC?ni>
z!LzYy{YIKAzHKV(XojRd<Uk!YH**rXE4coGr)Q_AX*J`A%dYvAe#zcZa68(ca3@j^
z6PT78tp_v18A@x@!t<`;N_ZAj`R&o2m?bB{vph{c#VgSd>TisnxBqC&m-6J{y_bu{
z0*iF><t0hD4UDZPe-_^1Z}F#I?B=f)*R9Eu)1YV{g;j(2Q89YC?kcKBplILh%_<k+
zA?0c0=+Jl3Il6A`ViF1cwP2&oE78>0-Q&&=>mNl~>b6bIpiJor!x3J-)+{x&31~tF
zHl()*|E(|0c1P{?)Bl`xPW&e$7o+hEszbAMjJD)%`jX!fbV6l*!P+<veG(!0!((4`
z#)RKJS1bZuES4Cb0ri}w(&ao)LRSrx(>y&G|N3=IjC%Qeoi}-p5$66wN!$QN8mlbm
zkT0Hj<mQzZt2uUxnX*sUR#z4WrWug=HX_foOuZzE!Om=N4}w?w`VcSqW3{Yr7xRlk
z))e|##h`$tk(}D?q2R4De2{J6Aeqp3DoAiv-^S;G4(ltX!?-{R&TGD&Cq)vjAw)3)
zUJ4%=k!;ftcHGE|!Qm)LK~0SE<w|>n8~%Mc3oml-of|S8RM?hEh?aSYlgTG7NYt~r
zm7IOu^I*#)^NUV+t&!@}O!l?~_0U87U`>Sb1s=-+wWc3+-Y>Y*lfznf<qCR+nR~8P
z;jU59kA5i|RUq*rEE_Fp<gb&+mJXQd87JjPMM+mXuvp?YZq(E8&VoWs$?eGjvyS>&
zX&kP~EXaoqS(u=<J!G^rWYgMS73fmN;U~`W!9~z#>o_uUe+&4*sMq57gX`NWkF4HE
z(=UcmL54Rso0&g5@Ql5^$!6cGc*Dkr><hNaY?iBFB<xCg%FdP(Z{rFH($|nOeU+M|
zpbYcD!@+J-HR8u~Hsw9n(by~9+&vt2z8VU}H4t9EbTZ;c=n5^;Oe-(C-}ShOQLeVB
z_T~~s#*59+f7rl9jPfKhDOXiZIyzMU#nhlWrv%y~pNy)ryWk;b!6a(q#<q)BUj0Mm
zgKedMLPNKaqRmC}dNKN?6$y)}Rgw2`s)n6YaX!VTa2~d9wrO_~BmCAYE}#j811oJK
z%=b~VKdKWkGSJqo(%GahvF~r`umQl-h+xuJLxP{6Cv5YrV{_4jZX<$M$0?icwj;l3
z3VJRUY??c3U3swS$N7Fk1=Hquo2{0X6oHjjXSm{#*n9ZT8hn*jNx<~=&PaCVcXM2H
zF~;Fk@yVOTRcyI-wdZFhT>C20NzQXTe&k5eb_B26nAD9}!O4(~8t3OTd1v2J@Fr{e
zJMGI;!}TXN-TX_xZ?c{8UN2AEwr?oCiP>QpI+x?&fbsAM^DZ{-QY@tQAJyLQ@dRgk
zR#|?jcy=k=dwpu%`dw@0%q;WZ_aXgY!sY&&<>Fg66?ZPv+hryxnpeE_@!m1^j;F19
zQGL_fa=p~U!Hu;x;zcZe!BpMXzD*%|Fnpofpw>-$y76I(_24bL*3}8Y8XnK?+9Rty
z9P@KI(*FAY+38=#zQq^BshXeaHp-0T|IhyZSXBpokVF0x{-yS%?pmTYc`to%%TiO{
zF@NJ>Kg(t_m?`j&ugnS)x^|fLwSZl<7wZ3<ok%+T$w~fK+y7xoo#}O<`m*L*_}k;&
z<)*|Oj+hj=9zHa>9jbU=-nK=YZ!gOMEgp!h_fs~I-VyVk|Mfdm7m4)lL1J@|UGCE_
zdqfK{r}&@WVGYM^^tTDxPY>SA@4u~740IVx`BtkPcttbAyy2Q5jPF{mGcydRH(vyL
ziT)MlWy_<mY@@+uqr&pT)yQ#LB4$9$)Q(_HAAkHP&qIHG#dFhb=i}omvSgO%2GQ2o
z(MP@mu`bOkxlLt*(M;DYydEl6)mX%G+-nhNsI|H@vG&%EQmA@);QI$QOz)$%!WTik
zFn;ay>6_{v9=&%KLBS8sI_v1)-(Qm$42_ChQ<Zt<V7sl$@x5t%_ShE8seyhFbKIT_
zXxH&(#S7QX*D?0H-@R=xA#ZQ5&s=<esANG#M|N`UNWO{QZyskRtE#j$*5%gAS5>L(
z%Yg=s`SOedRkp4U#Bb-d@xHmqeEM=+6~NV0tVMiNS$FI=M%pmw2odEf=QOf$m<TBx
z{OuXx3bpA^g)0ETfWjUiLY!;30l@;FsZ0mD@Rh{s;p+C!4uj<tYcotuy*vqSDu&NW
zkm#4*nO;ABWk^uV-S;r0Mh@0ap^J_#$pC!=j?FdG<%YR27543~Dw|kmulf$-<}&p*
z3zK0Io36`je$J@K9nlOx-t*N-7vv2tJFi#Gcy%cL>-X!Kdp46E)d*ZxraOK0wPU8!
zcoXlp9M#10_(#7g-vn<fF%H7CfeO<!vwl;?Ca6E?A8i6M#|IZKWLLEq9#eCW^cN2a
z@<1<$y^k+TIFK|Hw;6wZpdjtPe2%>gCZIslPK1C;sZS~&^UE1}RdxnyG8?(RQ_(-?
zfD^!B*qSBS^V%Cu87Nekz7;7ip+Y|ID^gq)Dc1feUXYPXAcDDtN+*;1MO}?1)=0Bb
z6CWkn<Bt>+9lGwJApINYG@Pja`9nfM7hb7cJuhHYoz_QzCNWN{P_ed~$tpJ}oY&<+
zv3!ac+bWbu<5u;2r+yeX6wnmh>3H6`r2f%h4*=c+Q&tej{3<JV>D_Sh$biHK=4;78
zv+nCWlD@Ttr#nK(depk@=o#xfY2>S8WPQKAB;jB3jV*6l=Zs^Xf-8&w3bfzONu;4+
zw0l%q8^y2Nla+d^RqyT4ca@KU`U<$(Ki^MWFA|pG6e`iRcLJ>ed2)&aq7W1nSjht4
zi1hWD<`z=R3-}O^5rA#M=Mumh^54;=J-3}wg_R6)V`!UZ%5kTbjADZh_hqy<Xycmh
zsI;>lp&?94W6|t<r6or+mn10-Cc&dxnY}lEte=u(8U4-twb33O`7}a`a`_x(5BhF$
zBzaT{wE<0@#Sw#pMFqoFJ)GbBnhG@>&(ODY+H^+xx(gf$wfP*HnzwA2y)MC?x?pmX
zy;50wNv4B7pwrt)fUm6UbNpB6+as+6IR*cV?(c99-?}KVfyL3_v@WqxSA9#*%Uw%$
zy4XP4;f2uCTMX&!^6n)ca{df$m<QE;+5Bu=Wo^r`J<JbIC1L!JP#~P$jm2tiIG!nm
z)mJbYM2v<WYFbso({(MqEd1s~H17Wr;~0>+EhK-HO0@BJr#3DK7{UZK3^R3mC_^@1
zvSwrU+$Q#N<^ve=piBF(>3%=<`Gf!V*Y|*QJ&j;Zly@USVZ&T%bCz<8sD0auOI>|r
z1`ad+yTKKEAnblG-~|NZkpIm~VvPV5R&e_{c~BtV=DT;_wmqqNLonMrce*)r2$i$0
zm|GWEJvN>$62Q_zR&Qo827629SLA2U!nOgmjT;F`HM%`lMl;zw9`$D&^bnj<6w5c9
z2MEbQ=9UT9hJaY?nO|)}Loh!pkf|gMP<TZTl@)ot0|EXR?=~#h+i=~S5x4N%X2AoE
zT!qw+`BaT-6eSM_y+bL4_ecz4geLDfaTF^x>NECNXAp?69gk=rlGjq?hG*$W1C=HE
zC4pSD9vBo<4Am@5u$@{}T=3$k(x^puy~%%fi-g-E<wDa)ftPE)N|pB#jj#S=hhLej
zA)(u}tMOfAHP9-MFzM{CI}qdn@*siYs&`u)ToKFz{J}cw8s~6SEqZKCQj53d(da<N
zPj21rCnN6n-CF>=Wt4%S{fk_oU$uWVK`!&xsaZP^%A0GM+iK`(u>2gUUA{n_)6~n{
zrhtl&fu&HedhBp#jQ42YMrx6Evc!fCX8Fp0eqP?vk6<?^<%m>lEHg%tU*Pbda%2U{
zD@HEz^L#E_>TCk3ubniNEFgYIjEY<*tOFy8!Tj2xA2FL6Kj<D&Ak}p#v^l<O$e$Sg
z(s%~V52`|7JP4$Yti?E)YumZUoc@mdX{BtmkSSD+<>NYapOd7?<c1P`OmLO}?(o;L
zIJoYaq`{WvY_LSXN}v-xx@RW49*E}6{yCxWya^xT9&?;P8jNyr#eCvH$C23hc!7Iq
zhevE`T<B}s0+kZ+l)AMl3;rfCP!ID5DaFYI`VnBU9A4wPkD-1AOF`Jk;9vx*!?&}s
z9ZGE>63u1gJ;<}?<yT;fh4(}iLTtvbcdNcV&lbE8ZyFzuiur8mRHqxj3mL^SjML}}
zzY+l{>Dd`B2Skj8Ys;o`6_Aw7Nw2J(<h_mx?ivNA22q3zoCl~GX!cN-4e321^sa0o
zzKafNC>^uNtSTcqkI4`NlMa6^Yt6T2iRBI9colUXF>3m`l9BTBFEm(JziHvHi+M|h
z)r%sJx#P!~ETXsrmnr-eF+1&oG#BAoK}4(r1-h2(X(?^Cgz%md$?=!?av3@mS*=8)
zfMy$pR^ep}$~o~_dnPk~s!@pK2fT)fNIdutyz}TSBmU<Afrsf3;_@Z#>flNP(B!^Z
z%|4p{?~ee$8op?I5Ihl%-}^f8iw6|ML98G`<@=4&>VQ(?EweYOmFrj{p8;<piOb{k
zAO3?#Sb#phUU|#2SEdBtuu+5zL-->sPjMUJ_Y(P{OdS7m0#Z)~wixMR<k~{$sW#ww
znB>sqTkgJu9^G<qU;B1v;huu%7=h?HGh#5lUEdHqu~T>7vqB1}mt1~7yz6UV>g#Q_
zR|Ev=TYFslL|x;SWE%rO@8IYF>t%mcHD9Ptdrlh!<fW&k;ytXLewHZu`n6$1GuPVQ
z7<;;g*R1bs26?ac-@A6!QkcDQF6Lr7cD2i}`21>q#bjJVy@HWnqzg_QH+tSB_}8Z$
zSe4uXUoLo(xkjR9&)1osMm|3uGH8`~^2SouPq?OLRS;+<ZX=?W=OTHMnCXzRsNL<L
z!{d!NRi44R<&%>tvIUEm>m8><_qZka-UM%=>tB4J=D+<$pN_D&zSK-J4oj~`70W#c
zZtW5s&;4$o=3oyPUBi&uOmnG2v=2uNGonWxwO^SKscyTjmAwmVu$$$*drh@Av-q`W
z)7flOsXJn&^UlAK9B#AG1bQMtpj|$R^jnr=IelMo^GA3&X%DR6Ug2T6UX|ohwGtxR
zv|W-An%U`dG+DLkf_c^%wj?cunk~_dRRW%g@b%ut$SG8?OXIzFUJKu6XM);zDSaDW
z+28vG7qN)sImmC^S4E{Cor*QDaUKuWG7qqs@5M+&o>(v#f5B#DWNJ>F>G1sA(?bX&
z3{WqeXD2O5&O{*2KceY^B&}jWRZye_?~xWw{50PvDKp|3#5SfnFbh{BWw2e>DLIBy
zW}N(ba)JEpI+JjMs=ajL#!8=EdC?nCWnz>ZD1&gBoI1(JH7xz~^-oII39GttQ#V7W
z<#YZKo1J9{r2&kbm0rI?(=|y@XL~=iW<jzc{L<}AutoLG%(qqa+E^~4Q8H#F&)m)m
z^+``UF|RTF0#nm51?qAn5N_qTCigMqK(b<d?@x2!xD)e4(IsX5HYfW;L}(#xT?Tcl
zD?f_7th4Qvd8f!n@lO)1<mS(vND6gn6g7YJo+f}?8(t6xOZ3B0<T!V2=a%x=MN|fD
zfq@^_@kUQ}Yxe<1$fi0oPi%n7giPQjg5R4ojKsr_o>__dC3MwFEX+<HM|Bqr4boHQ
z27_Wn@d|Eccj^fJ6=;XB=fidQPOfz1Yj3*KI8IFW;S^>U&m1QWvyx;p^HPa;tz*Tw
zp~}LSb;U?5$-(}t5V^rd!h5d#euj4Ejet(Ii%GP)U(hNene7i#QoMQ!1<4I&w0y;$
z2b`ax`l*@!mycrrn-uSdJ|XG3>Zsmxy2h=&D8YHo{a~=?46%6_%g5;!H`;d!#dSsM
zEWfxC1C9zJ@pos0{{DpG2Zi`CQIy&#z7X#ODb)GNlD>e%*~-XKiOUzrkV#zb2R-P6
z*qf9&LzlucGu#H)aFHY;jcHR!hXiIPvOihHN>REqZnPp(O`De7$#^rg>m*t)4~(9E
zXrg+8F=UA~`Lv?I4L_-uhrLvnrN|9BIuLDgb}Uc^%f`vxvqyZ=Eg;vFx`oP{rv5nb
zq$gk%!e$ivVG?MF#My5umm5VQDA11s^j3%}q7P+2pY&TlJ-H$90}w-HiY_T^M5|uu
zya*&bXlAQAWrI``qf^&z*c97JSbb_YV}EW-+K?rgAUkNKh*lLl`I2&x_f&RQLPK=4
z)x=thNBGJ3az?9h1lB|p;+_&x5(%y-!|N=cduMs4d&VHhvmMqt4fU7QcJ<~AZ^glT
z(fv&td{gj0jpP{Uwx?!;A~SSBy~vT?5>g<^zkZfe=~b`(HWZQi4oGg>Wtyg5Nzd4l
zV_%4NJdRCe6i#|fW0h9AVPe=2FPk8$K8T~!eIH|vxe(uFt@xEPi*ymqW|0-`ETT|0
z)!r)Cq)B&C0du~wl2Hv6{0a0jrAOKHdO&KZ{`T`M;IX3o(|iY1LY$PRMtw=0K9Qh>
z$68CK)DqV?i^;V<pc4<ucW_TnP2IYhzf<N_yY62L9^A0Wd%3@FCHRfv`Qbbv=F7em
zcnbFd{3&7(3^|NCmYi)QNZ$vgbm;%BKGBj?cm~x+g^cx7BNHK)hWtJOl~jT9J86@!
z_l)h8V3cFgp}`y3Kqf`vyBxV{Wn@ETt;;>BlCh-fXV2cNe0xci+{xsPf<t-dj#{b(
zt3zVVnem`Q?lLqIV~R~5X{g-?B(vY6QDzLLGoS>6>WehV>FZe(cbDi)@{YBdD#w7m
zf^?C)k?8c8a(pMRug`@nj=ZE8Sq!OyB1%~dNxXwTIYOn{8q+V)$;~`L+|lmBSjYZQ
zB`j*oqklgrR`;b?qHq`f&q&F~?nD}lk$i=!p)+SGgs*ZuR%W&#tYgqr=;PBl2&{-h
zSk@U7)e9(9;F+>}N{o`epy7k(SzO*w99lU95psR(mtcX)gXo=ho*BeR8v^|c<L>Z0
zgCVLeshbhsydb4`(S}ATcN&xWoZ1)uc22+1U$r+$RjmBm4iy{=u#{V0lahQP#9B)`
zg*KP&-I_n1#BuG@R2<3qiRX{6>qL^qgR-NwLcg7iBMRv*4(21mQ;*x8D(C|ZTPg1J
zG}~0H3w^F)V|*`U;A8UbOoryxY1-|S-)iqBtJ3UzG@skR8@$6<S!$$TiyM58BgT9i
z+7f3M8rymy%z|3%qaAA_iDX_JFC>nphY-wt4E!x5N0h4q%C5xOPzR!KF#AvIbT_>R
z`u1cp6v?B2JVf1#T$X|kOF>VutJK!<l&F(X;0_63yiyXE<2!s>9A+&Qr?Fntzernu
zK2)-c$9s8IXPC$Uc`Ru`uv=EpPwO+o6U<#zF~$Q-U{ZV59M{{LKe6;sL@7b#{gQjz
z!V@Krybk>*q{vE%@%VZg`gKtx2SH|r5&IKPEKeehNo~p}z*t<RAUr7av^Kf^4YvMT
z0|kAXLo2W@duVB`c*17JeN`7Y$ida3_4<;64%V9h>rzCSbG!QNCX&YK&B1N-k=j_Z
z>p*JVIi>DY_mr;79uN6xCAY{q49Ma(I;~CN<1SR@_U4I}N2`x@<rH)!w7#=_sJ&Ul
zmUY*xCg$1;norG*lm!_<%o9n5`lgpBMq1e><T70xat*3OT%w{sx0z;M1k2GHi@+<|
zZ`8+78lY~2gpMBG>8^qFJ=%3iLD3D=a-NbDFf%3MP!~%Q+KPiW_&kS;=q>tT^hyr+
z@&UmWX%DbCF=iIU|NZ^9YOiF=R841vQ$3KQ6SD2{+A1t7xk-EwpG(Y??92u~eW|ur
z!fe1p{78aF#eCw`eMe`+K~EdQqIo*UA~XqcKL9ron#TB?V&t?L)wW1Y4jYL|@mp`@
z&X6*2n$M`?sv6V014D(3s#q#Q=usAaX&}M+nI!7Cl4!!J2weG^R$*-GdSzm^Z{mP5
zV~W0$CRM7fIa@{|jD8JK{1~e72*%00N7GLY!ddA$hjMi&n`|FL`}L!acYJ<S+?jr3
zQ3)Ho!8U^vD|AS>YPEr&DJDi~(4_ZF0?tr7T`Rx)xlZ`3w70^RGKEP+Z)a;%`(f;h
z9?8!dOv{&@mKgS|GBsHwVU=;ZJguDC?XI1wbmA2mqedQZN9+nAHQ4zwu!?>XG`u?W
z_OJ;hm{^Ps6U&ow=*V%SFAufwntqX(CAX!!E+}%=zLomT9*H+f%mhm2mnok&VJ}YI
zkN~E$agXh`chTQpxJ*hK2v43KqYt1*s!9}`44zUAP#c|ap+HrZO}3gLe-+{LjvcaK
zQhZjhE8oa!20IllZpguvVZGG7_FK63g$o?Vdf8?hXe742);UF-BSrTr&yAl+Q!M#Q
zEQxNMlqT-hifBsi!jnK^njzRC_t8H}kbf#CgP($?Onvu7eWFadNT_n=c+i!ghPZXz
zG|Y{ab0nZPhSh1rD$}Xnc*V!qsd>O*ZeIU!E(od((gkiVn&57cSkd0LdGz=-3B4zX
zfE*?t)MNMO(_aL1d!c9Zo7#z@$x~hUQs3VUNJQ&e;S>DdT}F_IUY*YRr+%#yZaxqK
zsFEtVJ8jMZ+cQ6L9q>=&N(WvJiYi%JYv)YK*;4DE(u9Ky$TXO4>uUr951Nt9n$$JC
z<lL<mh3)Wq7M&f`S9a{Q32taQ_yH(#OgCt=4Gy?LjU5^)_w2EGQ=g)yL_YSH2UpPW
z$N1W;q!R5B+P_E@5Cb{)YRYnI>F3YfW*IqQkn;V66}gIfv(|NcKNa>IoIo!*U+sxV
za;g3vYib)%)Yv36*m-Z$`HjkQ(wuf81>2}Nal#N2J?oF8_}1~!8}V!HZDLBU0<E{X
z(j_O~sQfK9S(-s0%`TMJq_T|T7AOb-PfX*(Ud)eeV|qE!NS&7s^7?V&_=p8wU_W3Z
z6#rxfxOUlZ#m=c}aJi5oLZBx<FB|#BV0^|RD+8FXdRVsHq(?SV7ZeGfbWP{0^;(%l
z)8l5ZY6vfxbzVsfSXR}5tD8ftQKm(U4ne-_4Wz-VyUWYN^+nxN<pRLt{$1&#jogP3
zjS0SZz6rCDV(+)>Pj?sZ43_zBz2Eqrm%nAY{!sYaYg5^L`wgSYP?1AMaEVv&6%<Qz
z+ik9kO`dx4OU{t3ks^PA0f9#^+}suE=lToc&#ps{!`7bpv>m>cJ9kdxokvZGNG}NV
z(jl{h*c7v#*l>b<Tf2F>8GGej=wk}3Kak1sBLEGU+18f{`TDuC>aTrn!_kwG&`tdv
zUtc_?J~Ju-$x4dVbE^|%;~g!QC-2gyG0}Z*`*y`_z|m}Bq<KYXmzrCy#F`Q*zdmp|
zP&B%xd@a;>g!ew2Mgp6!i+}($4&>0NOhrdVXpE4auwpb2pg7*K5eks7*~D7)yTRRN
z{j5Rrhp?%X$n_KK-kMsGy>=dzWa@dUO^fRd<3SY$i{t%mp==AhaY$V$_RLol@rpgq
zdP|;k(miO(aQ(Q-qjgTZyz!wX2q3Ct1wKG}7*xBxwHa52a2)lReBHDdD(~iU*vCql
zVTtuVxt=-G*&|=eY}g>~na`sv$FPXpC3^+%^jYF2Yxpmm5#G6elttQ~ehk|^l|mAU
z=36o}*EeG~nqCs<ryMJiv6TxFpUuR{c%b2wAAg+GhE;2Or=c;h|0bJB>`0IUkVJC*
zDAf?O^i89uE#;iq=q@dfcy)8ea+7Aa@n*Ax`!U`kMf8dJ`s&)%*EUVgOi2Sf>v7XN
zr)qt4QWNcgic36{D)X$CcbD((hh9m9ps-n_!WNJ<1s>BG;=}Hp+A_5TyH(pRN9BmM
z=FOavhGoXAX_1(2Z>P64j+#`JEu@^^$+JlE<s2eqo+s)_B*?0D1$G11&4@6nxK)vS
zMQ=KFIdd2bhk+lrZ*p-}8+EyX{Bv8#!2p3V%9x#B_4kaT%&7&&u5N>et}Beu1Xyfh
z>pKIF*zbOIJ9}7(c0q1B?qQU$^lHO+)3)UH_+xI=kF{clHO%)J`Y8!qxbrY%!-M4d
zzWItvHQFL8>5$kgqTv^%M)r|;?a<U3?GqL-+|dQLG28y_4>|>iwgW6)C@%Ixav6i@
zX17-wULSIFB5}_U*G3eVewYrX6DIlUApXXe1}&bGUkng7iO!najiydY<MP6vxB3G}
zNI$sj>;&YYUB3V#fK2z6Vu_`?h2advv+FVGG{6o~L}{C5)pV$Dw{~Ybo8{uEoBKtV
z#6$(QteM0B%^ei%o;Ay!$(MX0m`g)r`2>>m%=247rOx~^7af^pZA5|2IEvCR63JKR
z(QbapI&`eyVoh_+BLl0mQlVi*hxE~QHB#3HE}R&>o-d2FvVCE8t&=#pU{nc4id-SI
zcxc8+F0P`!K_(aW5az37VL$K50%1rV6;c}fRqqa7Tpzbe(F8s{H=ehUWI*>=)wzbH
zQX2I`maaUI2__T$E%u#QNDFv+aQ$GzhFOjR6)|yB&47~LiB$n>Pv#{t!@skv+4#zo
zOqB0a(juRi7`(YPgwcQeJgP|0ds$oA(JFk!In(hfr&QwYW)bBCa`6NF=}*cjz#O4^
z-S0CJnZ;6YQVVN@4LHPMRVEVc3M`G>te!mhQF1hhgTR2~D2<*ioDuTQ++iM4XUPZ_
zdwSaO-UWVF$6J4k{ku_02)T6syo*?D(7?2N7R2_-5Ebp7Sk0sNIR|H}(|JdzYW%}Q
zie|*?5kII0GT&&b)(<hSzccXrNI{2C!zJ$%=O(J5H%bDV(vny@O)4i3A2t3A(~#h&
zX+WM0r*sF{&o(T2)&P&X<8^i6pk78eP7i6b2P?~WPL-dmC*SI8i*NG-#>=QWUiNpm
z@31Zwt2AT`Ts={^@W$zDk`zYfB77~6I?2=DmwY*$%id~)B19R%1dpHVqg(!f7-ZhP
ziPyDP0Io}9B>eNy@44D9v5IzLn4HfFPP&!xN0*5n=T5<_(&O&7XZqRt`-n=kYUN&3
zex**&@U?`v^#%R{&TuugYYZt~O`_d|y%6zUiEFxQ`n&oQJ2ays_eG3?sKGx#N1s))
z9+9Kiv@=U`H#GxAzN$6Zzg7FGB&+u{I1fw$V(-B18hi{)v$qfKPL34`>s2|=g<N#G
zojdtqAJK%Nv4OcnoJ_->=nuA*909oOeibn>$r!*`{wWRLSK{6Bhra|nsKMtsc}%Mq
zF6IWa55B@g5H$bt>_Zsm|F0j}!&`|s>E<v@JRg)o?>DiJ$MQac;{e>hKWJ<>{@Ug+
z6BBMDYWU%<FJ7`4Rx-AZH*Aj>TmSP{iCjA9jmKUK2Yfu-n^F0Ha5PEfFp^4q8%0l7
zZS_xvU9iA|Wu<dfZpXIpM_D9Lr4E9O8>mu;wzkVGY6}+YiiQFyd)!0z_$>fC9l_r6
z_g=EP*J>yCI@z@HEt}~mZ|`CIOdydh8hRya>GAK6lH?wAW&ixi1XfRbDhk!qOhf`n
zt)2HQL8UVK*0$YX5s|xKw)zN5VE^z-dnoY#+;{6*PdL&5lB=g~9vE>;GFZ2NX!Eq+
z^n&R^A}N<GLZs!0r<X`r9ITi9de(oRj=ezehc4Z_(*KoF0~bvFfM6pQyZ_^ZD3&v*
z*;tu3*qfm~ONM*J(qQ`Za8Mv^RO}jWh}{754UCXj77tIS{ox%1w*A5HX~z#ZR${;g
z(e!}6CW7zq!2f~86VC%YZ1^(4Z^6I%ACi!G_28HA==LfP6zPD}H%-d9q2(EDT@9P4
z)8i%H4Na9v0*z$w_<@c2nN66v%P50^iBa<eS>M~!Ifb`DpAH6slL6&ez?_}vT6j2Q
zEe8{IsxfJ~j%bEmQ%F$tw%!EFM@gk+n4>|&^mdhl)Gye%SI}4&WLa9Zja2tVM}XU|
z<J<qQ-AjJr{#A2Z%fWX*&Knq%U-O#J+G8~pf3<db#sO0{4pZRB;&179I#+#e!Df*Y
zg|?ZtXrXzHT6mHb9c$9#wp-odNfc^^XB&6|kAv5G<Am8ty6ynf0~3`o6;o+N+W#7i
zhM#y!8Um|)iDy>6Qu7oAHIY3O^I35TBC?S7AChY3i|)VzP9Q}TPbtH3TF^*uWfhrq
zA<!Kc8jC~e?lcw7!Rb>OWqc8hT81JlvT|13gb7h@RC=NsScZQVRjB)%h*i>JgHJ^j
zpy#AGJeuM=K_!Y=1{tmRu}55sjE&G!k$ZI4IExrhTq{E)zB(BX9wFUbV#ZgX4K$?<
zHeSGLQYf_=fn{Fpb1`PhuCEULWC!sRI_K2>(AMDc<4>^OCSOEr90g8z?gN63hgwL&
zA`TxXqwBnX&kP+|f-0jtYp(aaU*!iqitCFMo0&3-6zT2_syUdlfnUtt?1Kq^D&RPs
zwD_v{>LIeb90Un+qCcOFt<F?&!)fRq@CH;{EtIG1d8G4Ca?gW~ls_9MtyvT}uSCM1
z|9ny9BVG7TbKv{JJ|KY;*L(7uXvw@ksR_Jb^Zo$Qh`Bj*v&QlqH-kR=%m5V<p_cfr
zDW?wjQVm`2a6`1yA=jHPX({oJHO2SP_}bnwr?w%s#m50c>cJY|0DK(cG%!iV775el
zu99FO3y8f<_e&@xXKZiN3Xng=jA#2-W&@VZujRgfWs%$@x5063{GqRwviU+lBRsKW
zBAw9*IPpKB$s^`p3k%$_U^eVvVZkZnUeN_{o$Z?<?=SiPk)|i!1@^zU7gYW+DM6U}
z6Dj}?rUly>TfFmF#<US1S3zu>RRq4d9E6NN4#IGZIFKP7)NlQxW!!mAu)`)U<*wb)
zk?jow*IER#<RP+@&r#%lN;httJx+Uw>@bP0pYqxB$HNbOR3_&<q)dLt>Ql!`V)$7e
zZtnBPRhM0;|2|AcqN|ITo^ql2PNiH=ChITJ99Afjtk@db%kMt5N%L!P%DrMWaU2!N
zj17B258>+0OCu*xuHfm+)CQm77YGP`N1xh&k{=rT*T=ZP5%d#%=YpoX85H^6$DF|z
zrI1{dxy)E<1Gu$guPYsDHXYBZnPzW0EOPGDzKpgSBqM`l@(qoC%iiU*x~rcpj6k4U
zo13Ohn=zchxz(DXT1rszCZ$cJz3~Y60HsX;KlfSat9$tIXaR>Xu`(TG=0i;-xOMfJ
zF&=CuhWKv3LG>N?@U&OmTeFg&!QX7ybEZYr>KzqE`7`dc$DyA)bw(|da07V;K^rd)
zAA*)I-O4CukWxSMh2L%@79gvk)IAjz&%@K&nX!Z1Y|vLGLwYvD@0?bgppWYidgT6S
z<30FoulE~$KS&{u2L%mRuEui)Pq1K}>fr;a-bQzeKmIcO!D&SX`kxmE8qL3Z&fW+d
znE&#deoiOW104BrZn)@^de~?_3G_gFrCPxzHazXN<l*UO&ncnhu+>71j^sg`Btz5y
z0+N?Ub{0L=4x=y%z;LrbUybql@($VIX|HQqxrZ-KL+D%0%_Vk);c4GWSAKnwzD^DO
z+^q8$y~Cq^*;B@G=;Alf?ZHC?i{=E*W4~P`nz@-F;X{UM?T>lXNpl8Fg@+;ZZDy>#
z*WJ4Cw7TV{yQC1s_?T6lXOx`!jBof$`xsjYmwfk}e92FBy*ba@DG2C;EHbFyr1)ox
zfM;0Oc-eP)+GEh^$#$a#HbJkjuFgwCVxJ-OH&$$YtCh~tsI*%(4U?f*tWext(2&+8
zlcFtkz0o*J?$eO)qt(FG%*BiEO28q9f#$@9CHN?nJL4%qbXZq-KB<4j9cKiT8m6CJ
za7aUM3R9}%E?Ekpb^8y!9W^H7%Er?e$Bl>-f+N;!-*2CbNNZim@veGnMGn2_u+~ko
zVn{g^bxD0@$i{ZWqWkw_b#Q!OpjdLM8lxXpMQH6-7k_7Y2?2@4Jt1`LT%8sS414fh
zf4b~588j~mPd4znLrawQFf_(d^pX|Hz%`v#L&aQ9ypAdm#MXX6!!Q8DpsFsj(%W0z
zn%H|!iycDmI?n$XyUC!YF4Mq65>2>5dk7lyMdr_}TRx97<m8(EiHVr5`b-MVGcKMt
zCHz3C%Pjf}42wHKP>s?it5Q;lMn`+KO{rCX8Uk9QGAw;NCiy1pK}D_Mm*$H5?;(h}
zp&d-*#g~alF|O&{K|`TV@^Er!?D%Z$BDZ{rtwzTwgY@rHcUFyQ!O;LP*A|7j)i-|S
zBl1gk=kEoA1`30(nYCUDjJl@2vHT&DyL*{3oD>>6ZJ{&1Zh6Af>Re=+uU$_Kd&@@%
zl0xKO@#YrFtVL>YO|vCoYfQju3MYZa{<bh)&?ruRSCFQ0#`|3TO8JgYK6p%-h<nUP
z3z(5skz!5xyHQrs{BGJ93J78$d?QHeR}Bl;I|vqpHOya2Ux_rkb(0kO%3&yiv0<p(
z{Bo)MUUsC|S93z)Xf_G7yzc0IE6HwD@#pZHFpBGufnW}89HD`PBR-T4nBGs~9b(R#
z6Q~Z`SnD3GeqkJL?eWFz0fc_dnpfXCZ8>c$x7O7?XTs%Lxx#a-R{N64B9lIZJ~42J
zFd>jenRr1Un<FW$#%}j7JG~5i;@S}iXSPx+5g#>}KWrG!#WMb?D#5>^V}~?uHIoFI
zSGQ>!iO2l?fP+8tVexqdYlf!LIG?uWX`3TuX+js$V*Hmm^E-5A`73&wAJmzdKoIqI
z@-UdBzH9QfYUKkT%C^s#E!D=RIraJ&dCW`RuH4?9-cj`FK6MjBqS{30&mSDLb$DT5
VT7~Y*Gi&&u`W0>Eyi2IZ{|^|I@)ZC8

literal 0
HcmV?d00001

diff --git a/docs/learn/documentation/versioned/azure/eventhubs.md b/docs/learn/documentation/versioned/azure/eventhubs.md
new file mode 100644
index 0000000000..2defddb9d2
--- /dev/null
+++ b/docs/learn/documentation/versioned/azure/eventhubs.md
@@ -0,0 +1,197 @@
+---
+layout: page
+title: Connecting to Eventhubs
+---
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+You can configure your Samza jobs to process data from [Azure Eventhubs](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-features), Microsoft's data streaming service. An `event hub` is similar to a Kafka topic and can have multiple partitions with producers and consumers. Each message produced or consumed from an event hub is an instance of [EventData](https://docs.microsoft.com/en-us/java/api/com.microsoft.azure.eventhubs._event_data). 
+
+### Consuming from EventHubs:
+
+Samza's [EventHubSystemConsumer](https://github.com/apache/samza/blob/master/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubSystemConsumer.java) wraps the EventData into an [EventHubIncomingMessageEnvelope](https://github.com/apache/samza/blob/master/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubIncomingMessageEnvelope.java). The key of the message is set to the partition key of the EventData. The message is obtained from the EventData body. 
+
+To configure Samza to configure from EventHub streams: 
+
+```
+# define an event hub system factory with your identifier. eg: eh-system
+systems.eh-system.samza.factory=org.apache.samza.system.eventhub.EventHubSystemFactory
+
+# define your streams
+systems.eh-system.stream.list=input0, output0
+
+# define required properties for your streams
+systems.eh-system.streams.input0.eventhubs.namespace=YOUR-STREAM-NAMESPACE
+systems.eh-system.streams.input0.eventhubs.entitypath=YOUR-ENTITY-NAME
+systems.eh-system.streams.input0.eventhubs.sas.keyname=YOUR-SAS-KEY-NAME
+systems.eh-system.streams.input0.eventhubs.sas.token=YOUR-SAS-KEY-TOKEN
+
+systems.eh-system.streams.output0.eventhubs.namespace=YOUR-STREAM-NAMESPACE
+systems.eh-system.streams.output0.eventhubs.entitypath=YOUR-ENTITY-NAME
+systems.eh-system.streams.output0.eventhubs.sas.keyname=YOUR-SAS-KEY-NAME
+systems.eh-system.streams.output0.eventhubs.sas.token=YOUR-SAS-KEY-TOKEN
+```
+
+The tuple required to access the Eventhubs entity per stream must be provided, namely the fields `YOUR-STREAM-NAMESPACE`, `YOUR-ENTITY-NAME`, `YOUR-SAS-KEY-NAME`, `YOUR-SAS-KEY-TOKEN`.
+
+### Producing to EventHubs:
+
+Similarly, you can also configure your Samza job to write to EventHubs.  
+```
+OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(new SystemStream("eh-system", "output0"), key, message);
+collector.send(envelope);
+```
+
+Each [OutgoingMessageEnvelope](https://samza.apache.org/learn/documentation/latest/api/javadocs/org/apache/samza/system/OutgoingMessageEnvelope.html) is converted into an [EventData](https://docs.microsoft.com/en-us/java/api/com.microsoft.azure.eventhubs._event_data) instance whose body is set to the `message` in the envelope. Additionally, the `key` and the `produce timestamp` are set as properties in the EventData before sending it to EventHubs.
+
+### Advanced configuration:
+
+##### Producer partitioning: 
+
+The `partition.method` property determines how outgoing messages are partitioned. Valid values for this config are `EVENT_HUB_HASHING`, `PARTITION_KEY_AS_PARTITION` or `ROUND_ROBIN`. 
+
+`EVENT_HUB_HASHING`: By default, Samza computes the partition for an outgoing message based on the hash of its partition-key. This ensures that events with the same key are sent to the same partition. If this option is chosen, the partition key should be a string. If the partition key is not set, the key in the message is used for partitioning.
+
+`PARTITION_KEY_AS_PARTITION`: In this method, each message is sent to the partition specified by its partition key. This requires the partition key to be an integer. If the key is greater than the number of partitions, a modulo operation will be performed on the key. Similar to EVENT_HUB_HASHING, the key in the message is used if the partition key is not specified.
+
+`ROUND_ROBIN`: In this method, outgoing messages are distributed in a round-robin across all partitions. The key and the partition key in the message are ignored.
+
+![diagram-medium](/img/{{site.version}}/learn/documentation/azure/eventhub_send_methods.png)
+
+```
+systems.eh-system.partition.method = EVENT_HUB_HASHING
+```
+
+##### Consumer groups: 
+
+Eventhub supports a notion of [consumer groups](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-features#consumer-groups) which enable multiple applications have their own view of the event stream. Each event hub stream has a pre-defined consumer group named `$Default`. You can define your own consumer group for your job and configure a `eventhubs.consumer.group`  
+
+```
+systems.eh-system.streams.eh-input0.eventhubs.consumer.group = my-group
+```
+
+##### Serde: 
+
+By default, the messages from EventHubs are sent and received as byte arrays. You can configure a serializer and deserializer for your message by setting a value for `msg.serde` for your stream. 
+
+```
+streams.input0.samza.msg.serde = json
+streams.output0.samza.msg.serde = json
+```
+
+##### Consumer buffer size: 
+
+When the consumer reads a message from event hubs, it appends them to a shared producer-consumer buffer corresponding to its partition. This config determines the per-partition queue size. Setting a higher value for this config typically achieves a higher throughput at the expense of increased on-heap memory.
+
+```
+systems.eh-system.eventhubs.receive.queue.size = 10
+```
+
+For the list of all configs, check out the configuration table page [here](../jobs/configuration-table.html)
+
+### Azure Eventhubs Hello-Samza Example
+
+The [hello-samza](https://github.com/apache/samza-hello-samza) project contains an example of a high level job that consumes and produces to Eventhub using the Zookeeper deployment model.
+
+#### Get the Code
+
+Let's get started by cloning the hello-samza project
+
+```
+git clone https://git.apache.org/samza-hello-samza.git hello-samza
+cd hello-samza
+git checkout latest
+```
+
+The project comes up with numerous examples and for this tutorial, we will pick the Azure application.
+
+#### Setting up the Deployment Environment
+
+For our Azure application, we require [ZooKeeper](http://zookeeper.apache.org/). The hello-samza project comes with a script called "grid" to help with the environment setup
+
+```
+./bin/grid standalone
+```
+
+This command will download, install, and start ZooKeeper and Kafka. It will also check out the latest version of Samza and build it. All package files will be put in a sub-directory called "deploy" inside hello-samza's root folder.
+
+If you get a complaint that JAVA_HOME is not set, then you'll need to set it to the path where Java is installed on your system.
+
+
+#### Configuring the Azure application
+
+Here are the configs you must set before building the project. Configure these in the `src/main/config/azure-application-local-runner.properties` file.
+
+```
+# Add your EventHubs input stream credentials here
+systems.eventhubs.streams.input-stream.eventhubs.namespace=YOUR-STREAM-NAMESPACE
+systems.eventhubs.streams.input-stream.eventhubs.entitypath=YOUR-ENTITY-NAME
+systems.eventhubs.streams.input-stream.eventhubs.sas.keyname=YOUR-SAS-KEY-NAME
+systems.eventhubs.streams.input-stream.eventhubs.sas.token=YOUR-SAS-KEY-TOKEN
+
+# Add your EventHubs output stream credentials here
+systems.eventhubs.streams.output-stream.eventhubs.namespace=YOUR-STREAM-NAMESPACE
+systems.eventhubs.streams.output-stream.eventhubs.entitypath=YOUR-ENTITY-NAME
+systems.eventhubs.streams.output-stream.eventhubs.sas.keyname=YOUR-SAS-KEY-NAME
+systems.eventhubs.streams.output-stream.eventhubs.sas.token=YOUR-SAS-KEY-TOKEN
+```
+
+Optionally, you may also use the Azure Checkpoint Manager. Otherwise, comment out both these lines.
+
+```
+# Azure Table Checkpoint Manager
+task.checkpoint.factory=org.apache.samza.checkpoint.azure.AzureCheckpointManagerFactory
+azure.storage.connect=YOUR-STORAGE-ACCOUNT-CONNECTION-STRING
+```
+
+#### Building the Hello Samza Project
+
+With the environment setup complete, let us move on to building the hello-samza project. Execute the following commands:
+
+```
+mvn clean package
+mkdir -p deploy/samza
+tar -xvf ./target/hello-samza-0.14.0-SNAPSHOT-dist.tar.gz -C deploy/samza
+```
+
+We are now all set to deploy the application locally.
+
+#### Running the Azure application
+
+In order to run the application, we will use the *run-azure-application* script.
+
+```
+./deploy/samza/bin/run-azure-application.sh
+```
+
+The above command executes the helper script which invokes the *AzureZKLocalApplication* main class, which starts the *AzureApplication*. This application filters out the messages consumed without keys, prints them out and send them the configured output stream.
+
+The messages consumed should be printed in the following format:
+```
+Sending: 
+Received Key: <KEY>
+Received Message: <VALUE>
+```
+
+#### Shutdown
+
+This application can be shutdown by terminating the *run-azure-application* script.
+We can use the *grid* script to tear down the local environment ([Kafka](http://kafka.apache.org/) and [Zookeeper](http://zookeeper.apache.org/)).
+
+```
+bin/grid stop all
+```
\ No newline at end of file
diff --git a/docs/learn/documentation/versioned/index.html b/docs/learn/documentation/versioned/index.html
index e23b29f9d5..fa8e1e3e4a 100644
--- a/docs/learn/documentation/versioned/index.html
+++ b/docs/learn/documentation/versioned/index.html
@@ -94,6 +94,12 @@ <h4>Hadoop</h4>
   <li><a href="hadoop/producer.html">Writing to HDFS</a></li>
 </ul>
 
+<h4>Azure</h4>
+
+<ul class="documentation-list">
+  <li><a href="azure/eventhubs.html">Eventhubs Consumer/Producer</a></li>
+</ul>
+
 <h4>Operations</h4>
 
 <ul class="documentation-list">
diff --git a/docs/learn/documentation/versioned/jobs/configuration-table.html b/docs/learn/documentation/versioned/jobs/configuration-table.html
index ef81887b5d..6666bb1943 100644
--- a/docs/learn/documentation/versioned/jobs/configuration-table.html
+++ b/docs/learn/documentation/versioned/jobs/configuration-table.html
@@ -2248,6 +2248,93 @@ <h1>Samza Configuration Reference</h1>
                     <td class="default"></td>
                     <td class="description">Staging directory for storing partition description. By default (if not set by users) the value is inherited from "yarn.job.staging.directory" internally. The default value is typically good enough unless you want explicitly use a separate location.</td>
                 </tr>
+
+                <tr>
+                    <th colspan="3" class="section" id="eventhub">
+                        Using <a href="https://azure.microsoft.com/en-us/services/event-hubs/">EventHubs</a> for input and output streams<br>
+                        <span class="subtitle">
+                            (This section applies if you have set
+                            <a href="#systems-samza-factory" class="property">systems.*.samza.factory</a>
+                            <code>= org.apache.samza.system.eventhub.EventHubSystemFactory</code>)
+                        </span>
+                    </th>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-stream-list">systems.<span class="system">system-name</span>.<br>stream.list</td>
+                    <td class="default"></td>
+                    <td class="description">List of Samza <span class="stream">stream-ids</span> used for the Eventhub system</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-stream-namespace">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-id</span>.<br>eventhubs.namespace</td>
+                    <td class="default"></td>
+                    <td class="description">Namespace of the associated <span class="stream">stream-ids</span>. Required to access the Eventhubs entity per stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-stream-entity">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-id</span>.<br>eventhubs.entitypath</td>
+                    <td class="default"></td>
+                    <td class="description">Entity of the associated <span class="stream">stream-ids</span>. Required to access the Eventhubs entity per stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-stream-sas-keyname">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-id</span>.<br>eventhubs.sas.keyname</td>
+                    <td class="default"></td>
+                    <td class="description">SAS Keyname of the associated <span class="stream">stream-ids</span>. Required to access the Eventhubs entity per stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-stream-sas-token">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-id</span>.<br>eventhubs.sas.token</td>
+                    <td class="default"></td>
+                    <td class="description">SAS Token the associated <span class="stream">stream-ids</span>. Required to access the Eventhubs entity per stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-runtime-timeout">systems.<span class="system">system-name</span>.<br>eventhubs.runtime.info.timeout</td>
+                    <td class="default">60000</td>
+                    <td class="description">Timeout for fetching the runtime metadata from an Eventhub entity on startup in millis.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-send-partition-method">systems.<span class="system">system-name</span>.<br>eventhubs.partition.method</td>
+                    <td class="default"><code>EVENT_HUB_HASHING</code></td>
+                    <td class="description">
+                        Producer only config. Configure the method that the message is partitioned for the downstream Eventhub in one of the following ways:
+                        <dl>
+                            <dt><code>ROUND_ROBIN</code></dt>
+                            <dd>The message key and partition key are ignored and the message will be distributed in a round-robin fashion amongst all the partitions in the downstream EventHub.</dd>
+                            <dt><code>EVENT_HUB_HASHING</code></dt>
+                            <dd>Employs the hashing mechanism in EventHubs to determine, based on the key of the message, which partition the message should go. Using this method still ensures that all the events with the same key are sent to the same partition in the event hub. If this option is chosen, the partition key used for the hash should be a string. If the partition key is not set, the message key is used instead.</dd>
+                            <dt><code>PARTITION_KEY_AS_PARTITION</code></dt>
+                            <dd>Use the integer key specified by the partition key or key of the message to a specific partition on Eventhub. If the integer key is greater than the number of partitions in the destination Eventhub, a modulo operation will be performed to determine the resulting paritition. ie. if there are 6 partitions and the key is 9, the message will end up in partition 3. Similarly to EVENT_HUB_HASHING, if the partition key is not set the message key is used instead.</dd>
+                        </dl>
+                    </td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-send-key">systems.<span class="system">system-name</span>.<br>eventhubs.send.key</td>
+                    <td class="default">true</td>
+                    <td class="description">
+                        Producer only config. Sending each message key to the eventhub in the properties of the AMQP message. If the Samza Eventhub consumer is used, this field is used as the message key if the partition key is not present.
+                    </td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-consumer-group">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-id</span>.<br>eventhubs.consumer.group</td>
+                    <td class="default"><code>$Default</code></td>
+                    <td class="description">
+                        Consumer only config. Set the consumer group from the upstream Eventhub that the consumer is part of. Defaults to the <code>$Default</code> group that is initially present in all Eventhub entities (unless removed)
+                    </td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="eventhub-consumer-buffer-capacity">systems.<span class="system">system-name</span>.<br>eventhubs.receive.queue.size</td>
+                    <td class="default">100</td>
+                    <td class="description">
+                        Consumer only config. Per partition capacity of the eventhubs consumer buffer - the blocking queue used for storing messages. Larger buffer capacity typically leads to better throughput but consumes more memory.
+                    </td>
+                </tr>
             </tbody>
         </table>
     </body>

From 54dc4a03e0e1c506e7c73e58575e7c51898309a2 Mon Sep 17 00:00:00 2001
From: Wei Song <wsong@linkedin.com>
Date: Thu, 14 Dec 2017 12:04:34 -0800
Subject: [PATCH 14/36] Added document for table API to feature preview

Added document for table API to feature preview
 - Brief description of table
 - sendTo() operator for table
 - join() operator for stream-table-join

Author: Wei Song <wsong@linkedin.com>

Reviewers: Prateek Maheshwari <pmaheshwari@apache.org>

Closes #387 from weisong44/table-api-14
---
 docs/startup/preview/index.md                 | 68 ++++++++++++++++++-
 .../execution/JobGraphJsonGenerator.java      |  6 ++
 2 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/docs/startup/preview/index.md b/docs/startup/preview/index.md
index ceccc39dbc..20e4aafc56 100644
--- a/docs/startup/preview/index.md
+++ b/docs/startup/preview/index.md
@@ -273,7 +273,7 @@ The merge transform preserves the order of each MessageStream, so if message `m1
 
 As an alternative to the `merge` instance method, you also can use the [MessageStream#mergeAll](/learn/documentation/{{site.version}}/api/javadocs/org/apache/samza/operators/MessageStream.html#mergeAll-java.util.Collection-) static method to merge MessageStreams without operating on an initial stream.
 
-### SendTo
+### SendTo (stream)
 Sends all messages from this MessageStream to the provided OutputStream. You can specify the key and the value to be used for the outgoing message.
 
 {% highlight java %}
@@ -286,6 +286,16 @@ OutputStream<KV<String, String>> userRegions
 keyedPageViews.sendTo(userRegions);
 {% endhighlight %}
 
+### SendTo (table)
+
+Sends all messages from this MessageStream to the provided table, the expected message type is KV.
+
+{% highlight java %}
+  // Write a new message with memberId as the key and profile as the value to a table.
+  streamGraph.getInputStream("Profile", new NoOpSerde<Profile>())
+      .map(m -> KV.of(m.getMemberId(), m))
+      .sendTo(table);
+{% endhighlight %}
 
 ### Sink
 Allows sending messages from this MessageStream to an output system using the provided [SinkFunction](/learn/documentation/{{site.version}}/api/javadocs/org/apache/samza/operators/functions/SinkFunction.html).
@@ -302,9 +312,9 @@ pageViews.sink( (msg, collector, coordinator) -> {
 } )
 {% endhighlight %}
 
-### Join
+### Join (stream-stream)
 
-The Join operator joins messages from two MessageStreams using the provided pairwise [JoinFunction](/learn/documentation/{{site.version}}/api/javadocs/org/apache/samza/operators/functions/JoinFunction.html). Messages are joined when the keys extracted from messages from the first stream match keys extracted from messages in the second stream. Messages in each stream are retained for the provided ttl duration and join results are emitted as matches are found.
+The stream-stream Join operator joins messages from two MessageStreams using the provided pairwise [JoinFunction](/learn/documentation/{{site.version}}/api/javadocs/org/apache/samza/operators/functions/JoinFunction.html). Messages are joined when the keys extracted from messages from the first stream match keys extracted from messages in the second stream. Messages in each stream are retained for the provided ttl duration and join results are emitted as matches are found.
 
 {% highlight java %}
 // Joins a stream of OrderRecord with a stream of ShipmentRecord by orderId with a TTL of 20 minutes.
@@ -338,6 +348,38 @@ MessageStream<FulfilledOrderRecord> shippedOrders = orders.join(shipments, new O
 
 {% endhighlight %}
 
+### Join (stream-table)
+
+The stream-table Join operator joins messages from a MessageStream using the provided [StreamTableJoinFunction](/learn/documentation/{{site.version}}/api/javadocs/org/apache/samza/operators/functions/StreamTableJoinFunction.html). Messages from the input stream are joined with record in table using key extracted from input messages. The join function is invoked with both the message and the record. If a record is not found in the table, a null value is provided; the join function can choose to return null (inner join) or an output message (left outer join). For join to function properly, it is important to ensure the input stream and table are partitioned using the same key as this impacts the physical placement of data. 
+
+{% highlight java %}
+  streamGraph.getInputStream("PageView", new NoOpSerde<PageView>())
+      .partitionBy(PageView::getMemberId, v -> v, "p1")
+      .join(table, new PageViewToProfileJoinFunction())
+      ...
+{% endhighlight %}
+
+{% highlight java linenos %}
+public class PageViewToProfileJoinFunction implements StreamTableJoinFunction
+    <Integer, KV<Integer, PageView>, KV<Integer, Profile>, EnrichedPageView> {
+  @Override
+  public EnrichedPageView apply(KV<Integer, PageView> m, KV<Integer, Profile> r) {
+    return r != null ? 
+        new EnrichedPageView(...)
+      : null;
+  }
+
+  @Override
+  public Integer getMessageKey(KV<Integer, PageView> message) {
+    return message.getKey();
+  }
+
+  @Override
+  public Integer getRecordKey(KV<Integer, Profile> record) {
+    return record.getKey();
+  }
+}
+{% endhighlight %}
 
 ### Window
 
@@ -407,6 +449,26 @@ MessageStream<WindowPane<Void, Integer>> windowedStream =
 {% endhighlight %}
 
 
+### Table
+
+A Table represents a dataset that can be accessed by keys, and is one of the building blocks of the Samza high level API; the main motivation behind it is to support stream-table joins. The current K/V store is leveraged to provide backing store for local tables. More variations such as direct access and composite tables will be supported in the future. The usage of a table typically follows three steps:
+
+1. Create a table
+2. Populate the table using the sendTo() operator
+3. Join a stream with the table using the join() operator
+
+{% highlight java linenos %}
+final StreamApplication app = (streamGraph, cfg) -> {
+  Table<KV<Integer, Profile>> table = streamGraph.getTable(new InMemoryTableDescriptor("t1")
+      .withSerde(KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
+  ...
+};
+{% endhighlight %}
+
+Example above creates a TableDescriptor object, which contains all information about a table. The currently supported table types are [InMemoryTableDescriptor](https://github.com/apache/samza/blob/master/samza-kv-inmemory/src/main/java/org/apache/samza/storage/kv/inmemory/InMemoryTableDescriptor.java) and [RocksDbTableDescriptor](https://github.com/apache/samza/blob/master/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/rocksdb/RocksDbTableDescriptor.java). Notice the type of records in a table is KV, and [Serdes](https://samza.apache.org/learn/documentation/latest/container/serialization.html) for both key and value of records needs to be defined (line 4). Additional parameters can be added based on individual table types. 
+
+More details about step 2 and 3 can be found at operator section.
+
 ---
 
 ## Flexible Deployment Model
diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
index 2be01af5c4..48d22190f6 100644
--- a/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
+++ b/samza-core/src/main/java/org/apache/samza/execution/JobGraphJsonGenerator.java
@@ -28,6 +28,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+
 import org.apache.samza.config.ApplicationConfig;
 import org.apache.samza.operators.spec.JoinOperatorSpec;
 import org.apache.samza.operators.spec.OperatorSpec;
@@ -229,6 +230,11 @@ private Map<String, Object> operatorToMap(OperatorSpec spec) {
       map.put("tableId", tableSpec.getId());
     }
 
+    if (spec instanceof StreamTableJoinOperatorSpec) {
+      TableSpec tableSpec = ((StreamTableJoinOperatorSpec) spec).getTableSpec();
+      map.put("tableId", tableSpec.getId());
+    }
+
     if (spec instanceof JoinOperatorSpec) {
       map.put("ttlMs", ((JoinOperatorSpec) spec).getTtlMs());
     }

From ed8dad54bf04c3f81c2d686786722333325ca432 Mon Sep 17 00:00:00 2001
From: Aditya Toomula <atoomula@atoomula-ld1.linkedin.biz>
Date: Thu, 14 Dec 2017 15:33:29 -0800
Subject: [PATCH 15/36] SAMZA-1536; Adding docs for Kinesis consumer

Author: Aditya Toomula <atoomula@atoomula-ld1.linkedin.biz>

Reviewers: Jagadish <jagadish@apache.org>

Closes #384 from atoomula/kinesis-docs
---
 .../documentation/versioned/aws/kinesis.md    | 104 ++++++++++++++++++
 docs/learn/documentation/versioned/index.html |   6 +
 .../versioned/jobs/configuration-table.html   |  45 +++++++-
 3 files changed, 153 insertions(+), 2 deletions(-)
 create mode 100644 docs/learn/documentation/versioned/aws/kinesis.md

diff --git a/docs/learn/documentation/versioned/aws/kinesis.md b/docs/learn/documentation/versioned/aws/kinesis.md
new file mode 100644
index 0000000000..a4be3ddedb
--- /dev/null
+++ b/docs/learn/documentation/versioned/aws/kinesis.md
@@ -0,0 +1,104 @@
+---
+layout: page
+title: Connecting to Kinesis
+---
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+You can configure your Samza jobs to process data from [AWS Kinesis](https://aws.amazon.com/kinesis/data-streams), Amazon's data streaming service. A `Kinesis data stream` is similar to a Kafka topic and can have multiple partitions. Each message consumed from a Kinesis data stream is an instance of [Record](http://docs.aws.amazon.com/goto/WebAPI/kinesis-2013-12-02/Record).
+
+### Consuming from Kinesis:
+
+Samza's [KinesisSystemConsumer](https://github.com/apache/samza/blob/master/samza-aws/src/main/java/org/apache/samza/system/kinesis/consumer/KinesisSystemConsumer.java) wraps the Record into a [KinesisIncomingMessageEnvelope](https://github.com/apache/samza/blob/master/samza-aws/src/main/java/org/apache/samza/system/kinesis/consumer/KinesisIncomingMessageEnvelope.java). The key of the message is set to partition key of the Record. The message is obtained from the Record body.
+
+To configure Samza to consume from Kinesis streams:
+
+{% highlight jproperties %}
+# define a kinesis system factory with your identifier. eg: kinesis-system
+systems.kinesis-system.samza.factory=org.apache.samza.system.eventhub.KinesisSystemFactory
+
+# kinesis system consumer works with only AllSspToSingleTaskGrouperFactory
+job.systemstreampartition.grouper.factory=org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory
+
+# define your streams
+task.inputs=kinesis-system.input0
+
+# define required properties for your streams
+systems.kinesis-system.streams.input0.aws.region=YOUR-STREAM-REGION
+systems.kinesis-system.streams.input0.aws.accessKey=YOUR-ACCESS_KEY
+sensitive.systems.kinesis-system.streams.input0.aws.secretKey=YOUR-SECRET-KEY
+{% endhighlight %}
+
+The tuple required to access the Kinesis data stream must be provided, namely the fields `YOUR-STREAM-REGION`, `YOUR-ACCESS-KEY`, `YOUR-SECRET-KEY`.
+
+#### Advanced Configuration:
+
+##### AWS Client Configs:
+
+You can configure any [AWS client config](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ClientConfiguration.html) with the prefix `system.system-name.aws.clientConfig.*`
+{% highlight jproperties %}
+system.system-name.aws.clientConfig.CONFIG-NAME=CONFIG-VALUE
+{% endhighlight %}
+
+As an example, to set a proxy host and proxy port for the AWS Client:
+{% highlight jproperties %}
+systems.system-name.aws.clientConfig.ProxyHost=my-proxy-host.com
+systems.system-name.aws.clientConfig.ProxyPort=my-proxy-port
+{% endhighlight %}
+
+##### KCL Configs:
+
+Similarly, you can set any [Kinesis Client Library config](https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java) for a stream by configuring it under `systems.system-name.streams.stream-name.aws.kcl.*`
+{% highlight jproperties %}
+systems.system-name.streams.stream-name.aws.kcl.CONFIG-NAME=CONFIG-VALUE
+{% endhighlight %}
+
+As an example, to reset the checkpoint and set the starting position for a stream:
+{% highlight jproperties %}
+systems.kinesis-system.streams.input0.aws.kcl.TableName=my-app-table-name
+# set the starting position to either TRIM_HORIZON (oldest) or LATEST (latest)
+systems.kinesis-system.streams.input0.aws.kcl.InitialPositionInStream=my-start-position
+{% endhighlight %}
+
+#### Limitations
+
+The following limitations apply for Samza jobs consuming from Kinesis streams using the Samza consumer:
+* Stateful processing (eg: windows or joins) is not supported on Kinesis streams. However, you can accomplish this by chaining two Samza jobs where the first job reads from Kinesis and sends to Kafka while the second job processes the data from Kafka.
+* Kinesis streams cannot be configured as [bootstrap](https://samza.apache.org/learn/documentation/latest/container/streams.html) or [broadcast](https://samza.apache.org/learn/documentation/latest/container/samza-container.html) streams.
+* Kinesis streams must be used with the [AllSspToSingleTaskGrouperFactory](https://github.com/apache/samza/blob/master/samza-core/src/main/java/org/apache/samza/container/grouper/stream/AllSspToSingleTaskGrouperFactory.java). No other grouper is supported.
+* A Samza job that consumes from Kinesis cannot consume from any other input source. However, you can send your results to any destination (eg: Kafka, EventHubs), and have another Samza job consume them.
+
+### Producing to Kinesis:
+
+The KinesisSystemProducer for Samza is not yet implemented.
+
+### How to configure Samza job to consume from Kinesis data stream ?
+
+This tutorial uses [hello samza](../../../startup/hello-samza/{{site.version}}/) to illustrate running a Samza job on Yarn that consumes from Kinesis. We will use the [KinesisHelloSamza](https://github.com/apache/samza-hello-samza/blob/master/src/main/java/samza/examples/kinesis/KinesisHelloSamza.java) example.
+
+#### Update properties file
+
+Update the following properties in the kinesis-hello-samza.properties file:
+
+{% highlight jproperties %}
+task.inputs=kinesis.<kinesis-stream>
+systems.kinesis.streams.<kinesis-stream>.aws.region=<kinesis-stream-region>
+systems.kinesis.streams.<kinesis-stream>.aws.accessKey=<your-access-key>
+sensitive.systems.kinesis.streams.<kinesis-stream>.aws.region=<your-secret-key>
+{% endhighlight %}
+
+Now, you are ready to run your Samza application on Yarn as described [here](../../../startup/hello-samza/{{site.version}}/). Check the log file for messages read from your Kinesis stream.
diff --git a/docs/learn/documentation/versioned/index.html b/docs/learn/documentation/versioned/index.html
index fa8e1e3e4a..9b24c9cce3 100644
--- a/docs/learn/documentation/versioned/index.html
+++ b/docs/learn/documentation/versioned/index.html
@@ -100,6 +100,12 @@ <h4>Azure</h4>
   <li><a href="azure/eventhubs.html">Eventhubs Consumer/Producer</a></li>
 </ul>
 
+<h4>AWS</h4>
+
+<ul class="documentation-list">
+  <li><a href="aws/kinesis.html">Kinesis Consumer/Producer</a></li>
+</ul>
+
 <h4>Operations</h4>
 
 <ul class="documentation-list">
diff --git a/docs/learn/documentation/versioned/jobs/configuration-table.html b/docs/learn/documentation/versioned/jobs/configuration-table.html
index 6666bb1943..49886ce4d9 100644
--- a/docs/learn/documentation/versioned/jobs/configuration-table.html
+++ b/docs/learn/documentation/versioned/jobs/configuration-table.html
@@ -2170,7 +2170,7 @@ <h1>Samza Configuration Reference</h1>
                 </tr>
 
                 <tr>
-                    <th colspan="3" class="section" id="hdfs-system-producer"><a href="../hadoop/producer.html">Writing to HDFS</a></th>
+                    <th colspan="3" class="section" id="hdfs-system-producer"><a href="../hdfs/producer.html">Writing to HDFS</a></th>
                 </tr>
 
                 <tr>
@@ -2210,7 +2210,7 @@ <h1>Samza Configuration Reference</h1>
                 </tr>
 
                 <tr>
-                    <th colspan="3" class="section" id="hdfs-system-consumer"><a href="../hadoop/consumer.html">Reading from HDFS</a></th>
+                    <th colspan="3" class="section" id="hdfs-system-consumer"><a href="../hdfs/consumer.html">Reading from HDFS</a></th>
                 </tr>
 
                 <tr>
@@ -2335,6 +2335,47 @@ <h1>Samza Configuration Reference</h1>
                         Consumer only config. Per partition capacity of the eventhubs consumer buffer - the blocking queue used for storing messages. Larger buffer capacity typically leads to better throughput but consumes more memory.
                     </td>
                 </tr>
+
+                <tr>
+                    <th colspan="3" class="section" id="kinesis">
+                        Using <a href="https://aws.amazon.com/kinesis/">Kinesis</a> for input streams<br>
+                        <span class="subtitle">
+                            (This section applies if you have set
+                            <a href="#systems-samza-factory" class="property">systems.*.samza.factory</a>
+                            <code>= org.apache.samza.system.kinesis.KinesisSystemFactory</code>)
+                        </span>
+                    </th>
+                </tr>
+
+                <tr>
+                    <td class="property" id="kinesis-stream-region">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-name</span>.<br>aws.region</td>
+                    <td class="default"></td>
+                    <td class="description">Region of the associated <span class="stream">stream-name</span>. Required to access the Kinesis data stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="kinesis-stream-access-key">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-name</span>.<br>aws.accessKey</td>
+                    <td class="default"></td>
+                    <td class="description">AccessKey of the associated <span class="stream">stream-name</span>. Required to access Kinesis data stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="kinesis-stream-secret-key">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-name</span>.<br>aws.secretKey</td>
+                    <td class="default"></td>
+                    <td class="description">SecretKey of the associated <span class="stream">stream-name</span>. Required to access the Kinesis data stream.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="kinesis-stream-aws-kcl-configs">systems.<span class="system">system-name</span>.<br>streams.<span class="stream">stream-name</span>.<br>aws.kcl.*</td>
+                    <td class="default"></td>
+                    <td class="description"><a href="https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java">AWS Kinesis Client Library configuration</a> associated with the <span class="stream">stream-name</span>.</td>
+                </tr>
+
+                <tr>
+                    <td class="property" id="kinesis-system-aws-client-configs">systems.<span class="system">system-name</span>.<br>aws.clientConfig.*</td>
+                    <td class="default"></td>
+                    <td class="description"><a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ClientConfiguration.html">AWS ClientConfiguration</a> associated with the <span class="system">system-name</span>.</td>
+                </tr>
             </tbody>
         </table>
     </body>

From ea2b6fa915473e2f120a4860c32cd09b4516c283 Mon Sep 17 00:00:00 2001
From: Prateek Maheshwari <pmaheshw@linkedin.com>
Date: Mon, 18 Dec 2017 16:45:06 -0800
Subject: [PATCH 16/36] SAMZA-1538: Disabled Flaky Tests in TestStreamProcessor

Author: Prateek Maheshwari <pmaheshw@linkedin.com>

Reviewers: Shanthoosh Venkataraman <svenkata@linkedin.com>

Closes #389 from prateekm/disable-flaky-test
---
 .../apache/samza/test/processor/TestStreamProcessor.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java b/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
index c37132f9ed..a44265ab1f 100644
--- a/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
+++ b/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
@@ -67,7 +67,8 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
    * The standalone version in this test uses KafkaSystemFactory and it uses a SingleContainerGrouperFactory. Hence,
    * no matter how many tasks are present, it will always be run in a single processor instance. This simplifies testing
    */
-  @Test
+// TODO Fix in SAMZA-1538
+//  @Test
   public void testStreamProcessor() {
     final String testSystem = "test-system";
     final String inputTopic = "numbers";
@@ -88,7 +89,8 @@ public void testStreamProcessor() {
   /**
    * Should be able to create task instances from the provided task factory.
    */
-  @Test
+// TODO Fix in SAMZA-1538
+//  @Test
   public void testStreamProcessorWithStreamTaskFactory() {
     final String testSystem = "test-system";
     final String inputTopic = "numbers2";
@@ -107,7 +109,8 @@ public void testStreamProcessorWithStreamTaskFactory() {
   /**
    * Should be able to create task instances from the provided task factory.
    */
-  @Test
+//  TODO Fix in SAMZA-1538
+//  @Test
   public void testStreamProcessorWithAsyncStreamTaskFactory() {
     final String testSystem = "test-system";
     final String inputTopic = "numbers3";

From 7dc0290ad8babe3abb315bac2aa4155350a7053d Mon Sep 17 00:00:00 2001
From: Jacob Maes <jmakes@apache.org>
Date: Tue, 19 Dec 2017 08:00:56 -0800
Subject: [PATCH 17/36] SAMZA-1356: Improve monitoring for state restore

Author: Jacob Maes <jmakes@apache.org>
Author: Jacob Maes <jmaes@linkedin.com>

Reviewers: Jagadish <jvenkatr@linkedin.com>

Closes #241 from jmakes/samza-1356
---
 .../samza/storage/kv/KeyValueStorageEngine.scala | 10 +++++++---
 .../kv/KeyValueStorageEngineMetrics.scala        |  4 ++--
 .../storage/kv/TestKeyValueStorageEngine.scala   | 16 ++++++++++++++++
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala
index 646b60608c..c42e043563 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngine.scala
@@ -99,6 +99,8 @@ class KeyValueStorageEngine[K, V](
    * batching updates to underlying raw store to notAValidEvent wrapping functions for efficiency.
    */
   def restore(envelopes: java.util.Iterator[IncomingMessageEnvelope]) {
+    info("Restoring entries for store " + metrics.storeName)
+
     val batch = new java.util.ArrayList[Entry[Array[Byte], Array[Byte]]](batchSize)
 
     for (envelope <- envelopes.asScala) {
@@ -113,11 +115,11 @@ class KeyValueStorageEngine[K, V](
       }
 
       if (valBytes != null) {
-        metrics.restoredBytes.inc(valBytes.size)
+        metrics.restoredBytes.set(metrics.restoredBytes.getValue + valBytes.size)
       }
 
-      metrics.restoredBytes.inc(keyBytes.size)
-      metrics.restoredMessages.inc
+      metrics.restoredBytes.set(metrics.restoredBytes.getValue + keyBytes.size)
+      metrics.restoredMessages.set(metrics.restoredMessages.getValue + 1)
       count += 1
 
       if (count % 1000000 == 0) {
@@ -125,6 +127,8 @@ class KeyValueStorageEngine[K, V](
       }
     }
 
+    info(count + " total entries restored.")
+
     if (batch.size > 0) {
       rawStore.putAll(batch)
     }
diff --git a/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngineMetrics.scala b/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngineMetrics.scala
index 28cc891ecf..f43b52e1ff 100644
--- a/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngineMetrics.scala
+++ b/samza-kv/src/main/scala/org/apache/samza/storage/kv/KeyValueStorageEngineMetrics.scala
@@ -34,8 +34,8 @@ class KeyValueStorageEngineMetrics(
   val deletes = newCounter("deletes")
   val flushes = newCounter("flushes")
 
-  val restoredMessages = newCounter("messages-restored")
-  val restoredBytes = newCounter("messages-bytes")
+  val restoredMessages = newGauge("messages-restored", 0)
+  val restoredBytes = newGauge("messages-bytes", 0)
 
   val getNs = newTimer("get-ns")
   val putNs = newTimer("put-ns")
diff --git a/samza-kv/src/test/scala/org/apache/samza/storage/kv/TestKeyValueStorageEngine.scala b/samza-kv/src/test/scala/org/apache/samza/storage/kv/TestKeyValueStorageEngine.scala
index f6746855c8..ec714b2a85 100644
--- a/samza-kv/src/test/scala/org/apache/samza/storage/kv/TestKeyValueStorageEngine.scala
+++ b/samza-kv/src/test/scala/org/apache/samza/storage/kv/TestKeyValueStorageEngine.scala
@@ -21,7 +21,9 @@ package org.apache.samza.storage.kv
 
 import java.util.Arrays
 
+import org.apache.samza.Partition
 import org.apache.samza.storage.StoreProperties
+import org.apache.samza.system.{IncomingMessageEnvelope, SystemStreamPartition}
 import org.junit.Assert._
 import org.junit.{After, Before, Test}
 import org.mockito.Mockito._
@@ -129,6 +131,20 @@ class TestKeyValueStorageEngine {
     assertFalse("no next after iterating 2 keys in the range", iter.hasNext)
   }
 
+  @Test
+  def testRestoreMetrics(): Unit = {
+    val changelogSSP = new SystemStreamPartition("TestSystem", "TestStream", new Partition(0))
+    val changelogEntries = java.util.Arrays asList(
+      new IncomingMessageEnvelope(changelogSSP, "0", Array[Byte](1, 2), Array[Byte](3, 4, 5)),
+      new IncomingMessageEnvelope(changelogSSP, "1", Array[Byte](2, 3), Array[Byte](4, 5, 6)),
+      new IncomingMessageEnvelope(changelogSSP, "2", Array[Byte](3, 4), Array[Byte](5, 6, 7)))
+
+    engine.restore(changelogEntries.iterator())
+
+    assertEquals(3, metrics.restoredMessages.getValue)
+    assertEquals(15, metrics.restoredBytes.getValue) // 3 keys * 2 bytes/key +  3 msgs * 3 bytes/msg
+  }
+  
   def getNextTimestamp(): Long = {
     now += 1
     now

From e87165d403ac4ae66e5b0ee6b837420f0de635f0 Mon Sep 17 00:00:00 2001
From: Jacob Maes <jmakes@apache.org>
Date: Thu, 21 Dec 2017 10:43:47 -0800
Subject: [PATCH 18/36] Merge script improvement - use colons instead of
 semicolons

Author: Jacob Maes <jmakes@apache.org>

Reviewers: Xinyu Liu <xiliu@linkedin.com>,Jagadish <jvenkatr@linkedin.com>,Boris Shkolnik <boryas@apache.org>

Closes #391 from jmakes/merge-script-improvements
---
 bin/merge-pull-request.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/bin/merge-pull-request.py b/bin/merge-pull-request.py
index 5f1dae7176..a470045e1a 100755
--- a/bin/merge-pull-request.py
+++ b/bin/merge-pull-request.py
@@ -350,25 +350,25 @@ def get_version_json(version_str):
 
 def standardize_jira_ref(text):
     """
-    Standardize the jira reference commit message prefix to "PROJECT_NAME-XXX; Issue"
-    >>> standardize_jira_ref("%s-5954; Top by key" % CAPITALIZED_PROJECT_NAME)
-    'SAMZA-5954; Top by key'
-    >>> standardize_jira_ref("%s-5821; ParquetRelation2 CTAS should check if delete is successful" % PROJECT_NAME)
-    'SAMZA-5821; ParquetRelation2 CTAS should check if delete is successful'
+    Standardize the jira reference commit message prefix to "PROJECT_NAME-XXX: Issue"
+    >>> standardize_jira_ref("%s-5954: Top by key" % CAPITALIZED_PROJECT_NAME)
+    'SAMZA-5954: Top by key'
+    >>> standardize_jira_ref("%s-5821: ParquetRelation2 CTAS should check if delete is successful" % PROJECT_NAME)
+    'SAMZA-5821: ParquetRelation2 CTAS should check if delete is successful'
     >>> standardize_jira_ref("%s-4123 [WIP] Show new dependencies added in pull requests" % PROJECT_NAME)
-    'SAMZA-4123; [WIP] Show new dependencies added in pull requests'
+    'SAMZA-4123: [WIP] Show new dependencies added in pull requests'
     >>> standardize_jira_ref("%s  5954: Top by key" % PROJECT_NAME)
-    'SAMZA-5954; Top by key'
+    'SAMZA-5954: Top by key'
     >>> standardize_jira_ref("%s-979 a LRU scheduler for load balancing in TaskSchedulerImpl" % PROJECT_NAME)
-    'SAMZA-979; a LRU scheduler for load balancing in TaskSchedulerImpl'
+    'SAMZA-979: a LRU scheduler for load balancing in TaskSchedulerImpl'
     >>> standardize_jira_ref("%s-1094 Support MiMa for reporting binary compatibility across versions." % CAPITALIZED_PROJECT_NAME)
-    'SAMZA-1094; Support MiMa for reporting binary compatibility across versions.'
-    >>> standardize_jira_ref("[WIP] %s-1146; Vagrant support" % CAPITALIZED_PROJECT_NAME)
-    'SAMZA-1146; [WIP] Vagrant support'
+    'SAMZA-1094: Support MiMa for reporting binary compatibility across versions.'
+    >>> standardize_jira_ref("[WIP] %s-1146: Vagrant support" % CAPITALIZED_PROJECT_NAME)
+    'SAMZA-1146: [WIP] Vagrant support'
     >>> standardize_jira_ref("%s-1032. If Yarn app fails before registering, app master stays aroun..." % PROJECT_NAME)
-    'SAMZA-1032; If Yarn app fails before registering, app master stays aroun...'
+    'SAMZA-1032: If Yarn app fails before registering, app master stays aroun...'
     >>> standardize_jira_ref("%s-6250 %s-6146 %s-5911: Types are now reserved words in DDL parser." % (PROJECT_NAME, PROJECT_NAME, CAPITALIZED_PROJECT_NAME))
-    'SAMZA-6250 SAMZA-6146 SAMZA-5911; Types are now reserved words in DDL parser.'
+    'SAMZA-6250 SAMZA-6146 SAMZA-5911: Types are now reserved words in DDL parser.'
     >>> standardize_jira_ref("Additional information for users building from source code")
     'Additional information for users building from source code'
 
@@ -400,7 +400,7 @@ def standardize_jira_ref(text):
     # Assemble full text (JIRA ref(s), module(s), remaining text)
     jira_prefix = ' '.join(jira_refs).strip()
     if jira_prefix:
-        jira_prefix = jira_prefix + "; "
+        jira_prefix = jira_prefix + ": "
     clean_text = jira_prefix + ' '.join(components).strip() + " " + text.strip()
 
     # Replace multiple spaces with a single space, e.g. if no jira refs and/or components were included

From 29ecae891a2c9158b59e81d060234504d6844f70 Mon Sep 17 00:00:00 2001
From: Jacob Maes <jmakes@apache.org>
Date: Thu, 21 Dec 2017 11:43:09 -0800
Subject: [PATCH 19/36] =?UTF-8?q?SAMZA-1537:=20StreamAppender=20can=20dead?=
 =?UTF-8?q?lock=20due=20to=20locks=20held=20by=20Kafka=20an=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…d Log4j

Author: Jacob Maes <jmakes@apache.org>

Reviewers: Jagadish <jvenkatr@linkedin.com>,Yi Pan (Data Infrastructure) <nickpan47@gmail.com>

Closes #388 from jmakes/async-stream-appender
---
 .../samza/logging/log4j/StreamAppender.java   | 109 +++++++++++--
 .../logging/log4j/StreamAppenderMetrics.java  |  43 ++++++
 .../logging/log4j/MockSystemProducer.java     |  12 +-
 .../logging/log4j/TestStreamAppender.java     | 144 ++++++++++++++----
 4 files changed, 270 insertions(+), 38 deletions(-)
 create mode 100644 samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppenderMetrics.java

diff --git a/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java b/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java
index 84368353f1..0ea8b681c9 100644
--- a/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java
+++ b/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java
@@ -20,9 +20,13 @@
 package org.apache.samza.logging.log4j;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Logger;
 import org.apache.log4j.spi.LoggingEvent;
@@ -34,6 +38,7 @@
 import org.apache.samza.coordinator.JobModelManager;
 import org.apache.samza.job.model.JobModel;
 import org.apache.samza.logging.log4j.serializers.LoggingEventJsonSerdeFactory;
+import org.apache.samza.metrics.MetricsRegistry;
 import org.apache.samza.metrics.MetricsRegistryMap;
 import org.apache.samza.serializers.Serde;
 import org.apache.samza.serializers.SerdeFactory;
@@ -54,6 +59,12 @@ public class StreamAppender extends AppenderSkeleton {
   private static final String JAVA_OPTS_CONTAINER_NAME = "samza.container.name";
   private static final String JOB_COORDINATOR_TAG = "samza-job-coordinator";
   private static final String SOURCE = "log4j-log";
+
+  protected static final int DEFAULT_QUEUE_SIZE = 100;
+  private static final long DEFAULT_QUEUE_TIMEOUT_S = 2; // Abitrary choice
+
+  protected static volatile boolean systemInitialized = false;
+
   private Config config = null;
   private SystemStream systemStream = null;
   private SystemProducer systemProducer = null;
@@ -62,7 +73,12 @@ public class StreamAppender extends AppenderSkeleton {
   private boolean isApplicationMaster = false;
   private Serde<LoggingEvent> serde = null;
   private Logger log = Logger.getLogger(StreamAppender.class);
-  protected static volatile boolean systemInitialized = false;
+  protected StreamAppenderMetrics metrics;
+
+  private final BlockingQueue<byte[]> logQueue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_SIZE);
+  protected long queueTimeoutS = DEFAULT_QUEUE_TIMEOUT_S;
+
+  private Thread transferThread;
 
   /**
    * used to detect if this thread is called recursively
@@ -87,6 +103,7 @@ public void activateOptions() {
           ". This is used as the key for the log appender, so can't proceed.");
     }
     key = containerName; // use the container name as the key for the logs
+
     // StreamAppender has to wait until the JobCoordinator is up when the log is in the AM
     if (isApplicationMaster) {
       systemInitialized = false;
@@ -110,9 +127,32 @@ public void append(LoggingEvent event) {
             log.trace("Waiting for the JobCoordinator to be instantiated...");
           }
         } else {
-          OutgoingMessageEnvelope outgoingMessageEnvelope =
-              new OutgoingMessageEnvelope(systemStream, key.getBytes("UTF-8"), serde.toBytes(subLog(event)));
-          systemProducer.send(SOURCE, outgoingMessageEnvelope);
+          // Serialize the event before adding to the queue to leverage the caller thread
+          // and ensure that the transferThread can keep up.
+          if (!logQueue.offer(serde.toBytes(subLog(event)), queueTimeoutS, TimeUnit.SECONDS)) {
+            // Do NOT retry adding to the queue. Dropping the event allows us to alleviate the unlikely
+            // possibility of a deadlock, which can arise due to a circular dependency between the SystemProducer
+            // which is used for StreamAppender and the log, which uses StreamAppender. Any locks held in the callstack
+            // of those two code paths can cause a deadlock. Dropping the event allows us to proceed.
+
+            // Scenario:
+            // T1: holds L1 and is waiting for L2
+            // T2: holds L2 and is waiting to produce to BQ1 which is drained by T3 (SystemProducer) which is waiting for L1
+
+            // This has happened due to locks in Kafka and log4j (see SAMZA-1537), which are both out of our control,
+            // so dropping events in the StreamAppender is our best recourse.
+
+            // Drain the queue instead of dropping one message just to reduce the frequency of warn logs above.
+            int messagesDropped = logQueue.drainTo(new ArrayList<>()) + 1; // +1 because of the current log event
+            log.warn(String.format("Exceeded timeout %ss while trying to log to %s. Dropping %d log messages.",
+                queueTimeoutS,
+                systemStream.toString(),
+                messagesDropped));
+
+            // Emit a metric which can be monitored to ensure it doesn't happen often.
+            metrics.logMessagesDropped.inc(messagesDropped);
+          }
+          metrics.bufferFillPct.set(Math.round(100 * logQueue.size() / DEFAULT_QUEUE_SIZE));
         }
       } catch (Exception e) {
         System.err.println("[StreamAppender] Error sending log message:");
@@ -120,6 +160,8 @@ public void append(LoggingEvent event) {
       } finally {
         recursiveCall.set(false);
       }
+    } else if (metrics != null) { // setupSystem() may not have been invoked yet so metrics can be null here.
+      metrics.recursiveCalls.inc();
     }
   }
 
@@ -142,6 +184,13 @@ public void close() {
     log.info("Shutting down the StreamAppender...");
     if (!this.closed) {
       this.closed = true;
+      transferThread.interrupt();
+      try {
+        transferThread.join();
+      } catch (InterruptedException e) {
+        log.error("Interrupted while waiting for sink thread to finish.", e);
+      }
+
       flushSystemProducer();
       if (systemProducer !=  null) {
         systemProducer.stop();
@@ -169,7 +218,7 @@ public void flushSystemProducer() {
    * @return Config the config of this container
    */
   protected Config getConfig() {
-    Config config = null;
+    Config config;
 
     try {
       if (isApplicationMaster) {
@@ -196,6 +245,10 @@ protected void setupSystem() {
       streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
     }
 
+    // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
+    MetricsRegistry metricsRegistry = new MetricsRegistryMap();
+    metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);
+
     String systemName = log4jSystemConfig.getSystemName();
     String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName);
     if (systemFactoryName != null) {
@@ -206,13 +259,51 @@ protected void setupSystem() {
 
     setSerde(log4jSystemConfig, systemName, streamName);
 
-    systemProducer = systemFactory.getProducer(systemName, config, new MetricsRegistryMap());
+    systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
     systemStream = new SystemStream(systemName, streamName);
     systemProducer.register(SOURCE);
     systemProducer.start();
 
     log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
         + " in " + systemName + ". Logs are partitioned by " + key);
+
+    startTransferThread();
+  }
+
+  private void startTransferThread() {
+
+    try {
+      // Serialize the key once, since we will use it for every event.
+      final byte[] keyBytes = key.getBytes("UTF-8");
+
+      Runnable transferFromQueueToSystem = () -> {
+        while (!Thread.currentThread().isInterrupted()) {
+          try {
+            byte[] serializedLogEvent = logQueue.take();
+
+            OutgoingMessageEnvelope outgoingMessageEnvelope =
+                new OutgoingMessageEnvelope(systemStream, keyBytes, serializedLogEvent);
+            systemProducer.send(SOURCE, outgoingMessageEnvelope);
+
+          } catch (InterruptedException e) {
+            // Preserve the interrupted status for the loop condition.
+            Thread.currentThread().interrupt();
+          } catch (Throwable t) {
+            log.error("Error sending StreamAppender event to SystemProducer", t);
+          }
+        }
+      };
+
+      transferThread = new Thread(transferFromQueueToSystem);
+      transferThread.setDaemon(true);
+      transferThread.setName("Samza StreamAppender Producer " + transferThread.getName());
+      transferThread.start();
+
+    } catch (UnsupportedEncodingException e) {
+      throw new SamzaException(String.format(
+          "Container name: %s could not be encoded to bytes. StreamAppender cannot proceed.", key),
+          e);
+    }
   }
 
   protected static String getStreamName(String jobName, String jobId) {
@@ -228,7 +319,7 @@ protected static String getStreamName(String jobName, String jobId) {
 
   /**
    * set the serde for this appender. It looks for the stream serde first, then system serde.
-   * If still can not get the serde, throws exceptions. 
+   * If still can not get the serde, throws exceptions.
    *
    * @param log4jSystemConfig log4jSystemConfig for this appender
    * @param systemName name of the system
@@ -254,7 +345,7 @@ private void setSerde(Log4jSystemConfig log4jSystemConfig, String systemName, St
 
   /**
    * Returns the serde that is being used for the stream appender.
-   * 
+   *
    * @return The Serde&lt;LoggingEvent&gt; that the appender is using.
    */
   public Serde<LoggingEvent> getSerde() {
diff --git a/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppenderMetrics.java b/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppenderMetrics.java
new file mode 100644
index 0000000000..0273cb5e66
--- /dev/null
+++ b/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppenderMetrics.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.samza.logging.log4j;
+
+import org.apache.samza.metrics.Counter;
+import org.apache.samza.metrics.Gauge;
+import org.apache.samza.metrics.MetricsBase;
+import org.apache.samza.metrics.MetricsRegistry;
+
+
+public class StreamAppenderMetrics extends MetricsBase {
+  /** The percentage of the log queue capacity that is currently filled with messages from 0 to 100. */
+  public final Gauge<Integer> bufferFillPct;
+
+  /** The number of recursive calls to the StreamAppender. These events will not be logged. */
+  public final Counter recursiveCalls;
+
+  /** The number of log messages dropped e.g. because of buffer overflow. Does not include recursive calls. */
+  public final Counter logMessagesDropped;
+
+  public StreamAppenderMetrics(String prefix, MetricsRegistry registry) {
+    super(prefix, registry);
+    bufferFillPct = newGauge("buffer-fill-percent", 0);
+    recursiveCalls = newCounter("recursive-calls");
+    logMessagesDropped = newCounter("log-messages-dropped");
+  }
+}
diff --git a/samza-log4j/src/test/java/org/apache/samza/logging/log4j/MockSystemProducer.java b/samza-log4j/src/test/java/org/apache/samza/logging/log4j/MockSystemProducer.java
index 8d9909416c..fc8de0ba6e 100644
--- a/samza-log4j/src/test/java/org/apache/samza/logging/log4j/MockSystemProducer.java
+++ b/samza-log4j/src/test/java/org/apache/samza/logging/log4j/MockSystemProducer.java
@@ -21,13 +21,15 @@
 
 import java.util.ArrayList;
 
+import java.util.List;
 import org.apache.log4j.Logger;
 import org.apache.samza.system.OutgoingMessageEnvelope;
 import org.apache.samza.system.SystemProducer;
 
 public class MockSystemProducer implements SystemProducer {
-  static public ArrayList<Object> messagesReceived = new ArrayList<Object>();
-  static private Logger log = Logger.getLogger(MockSystemProducer.class);
+  public static ArrayList<Object> messagesReceived = new ArrayList<>();
+  private static Logger log = Logger.getLogger(MockSystemProducer.class);
+  public static List<MockSystemProducerListener> listeners = new ArrayList<>();
 
   @Override
   public void start() {
@@ -45,9 +47,15 @@ public void register(String source) {
   @Override
   public void send(String source, OutgoingMessageEnvelope envelope) {
     messagesReceived.add(envelope.getMessage());
+
+    listeners.forEach((listener) -> listener.onSend(source, envelope));
   }
 
   @Override
   public void flush(String source) {
   }
+
+  public interface MockSystemProducerListener {
+    void onSend(String source, OutgoingMessageEnvelope envelope);
+  }
 }
diff --git a/samza-log4j/src/test/java/org/apache/samza/logging/log4j/TestStreamAppender.java b/samza-log4j/src/test/java/org/apache/samza/logging/log4j/TestStreamAppender.java
index 59669b7dee..d93c5d1092 100644
--- a/samza-log4j/src/test/java/org/apache/samza/logging/log4j/TestStreamAppender.java
+++ b/samza-log4j/src/test/java/org/apache/samza/logging/log4j/TestStreamAppender.java
@@ -21,9 +21,14 @@
 
 import static org.junit.Assert.*;
 
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import org.apache.log4j.Logger;
 import org.apache.log4j.PatternLayout;
 import org.apache.samza.config.Config;
@@ -31,12 +36,20 @@
 import org.apache.samza.logging.log4j.serializers.LoggingEventJsonSerde;
 import org.apache.samza.logging.log4j.serializers.LoggingEventStringSerde;
 import org.apache.samza.logging.log4j.serializers.LoggingEventStringSerdeFactory;
+import org.junit.After;
 import org.junit.Test;
 
 public class TestStreamAppender {
 
   static Logger log = Logger.getLogger(TestStreamAppender.class);
 
+  @After
+  public void tearDown() {
+    log.removeAllAppenders();
+    MockSystemProducer.listeners.clear();
+    MockSystemProducer.messagesReceived.clear();
+  }
+
   @Test
   public void testDefaultSerde() {
     System.setProperty("samza.container.name", "samza-container-1");
@@ -70,7 +83,7 @@ public void testNonDefaultSerde() {
   }
 
   @Test
-  public void testSystemProducerAppenderInContainer() {
+  public void testSystemProducerAppenderInContainer() throws InterruptedException {
     System.setProperty("samza.container.name", "samza-container-1");
 
     MockSystemProducerAppender systemProducerAppender = new MockSystemProducerAppender();
@@ -79,22 +92,13 @@ public void testSystemProducerAppenderInContainer() {
     systemProducerAppender.setLayout(layout);
     systemProducerAppender.activateOptions();
     log.addAppender(systemProducerAppender);
-    log.info("testing");
-    log.info("testing2");
-
-    systemProducerAppender.flushSystemProducer();
-
-    assertEquals(2, MockSystemProducer.messagesReceived.size());
-    assertTrue(new String((byte[]) MockSystemProducer.messagesReceived.get(0)).contains("\"message\":\"testing\""));
-    assertTrue(new String((byte[]) MockSystemProducer.messagesReceived.get(1)).contains("\"message\":\"testing2\""));
 
-    // reset
-    log.removeAllAppenders();
-    MockSystemProducer.messagesReceived.clear();
+    List<String> messages = Lists.newArrayList("testing1", "testing2");
+    logAndVerifyMessages(messages);
   }
 
   @Test
-  public void testSystemProducerAppenderInAM() {
+  public void testSystemProducerAppenderInAM() throws InterruptedException {
     System.setProperty("samza.container.name", "samza-job-coordinator");
 
     MockSystemProducerAppender systemProducerAppender = new MockSystemProducerAppender();
@@ -104,26 +108,112 @@ public void testSystemProducerAppenderInAM() {
     systemProducerAppender.activateOptions();
     log.addAppender(systemProducerAppender);
 
-    log.info("no-received");
-    systemProducerAppender.flushSystemProducer();
-    // it should not receive anything because the system is not setup
-    assertEquals(0, MockSystemProducer.messagesReceived.size());
+    log.info("no-received"); // System isn't initialized yet, so this message should be dropped
 
     systemProducerAppender.setupSystem();
     MockSystemProducerAppender.systemInitialized = true;
 
-    log.info("testing3");
-    log.info("testing4");
-    systemProducerAppender.flushSystemProducer();
+    List<String> messages = Lists.newArrayList("testing3", "testing4");
+    logAndVerifyMessages(messages);
+  }
 
-    // be able to received msgs now
-    assertEquals(2, MockSystemProducer.messagesReceived.size());
-    assertTrue(new String((byte[]) MockSystemProducer.messagesReceived.get(0)).contains("\"message\":\"testing3\""));
-    assertTrue(new String((byte[]) MockSystemProducer.messagesReceived.get(1)).contains("\"message\":\"testing4\""));
+  @Test
+  public void testExceptionsDoNotKillTransferThread() throws InterruptedException {
+    System.setProperty("samza.container.name", "samza-container-1");
 
-    // reset
-    log.removeAllAppenders();
-    MockSystemProducer.messagesReceived.clear();
+    MockSystemProducerAppender systemProducerAppender = new MockSystemProducerAppender();
+    PatternLayout layout = new PatternLayout();
+    layout.setConversionPattern("%m");
+    systemProducerAppender.setLayout(layout);
+    systemProducerAppender.activateOptions();
+    log.addAppender(systemProducerAppender);
+
+    List<String> messages = Lists.newArrayList("testing5", "testing6", "testing7");
+
+    // Set up latch
+    final CountDownLatch allMessagesSent = new CountDownLatch(messages.size());
+    MockSystemProducer.listeners.add((source, envelope) -> {
+        allMessagesSent.countDown();
+        if (allMessagesSent.getCount() == messages.size() - 1) {
+          throw new RuntimeException(); // Throw on the first message
+        }
+      });
+
+    // Log the messages
+    messages.forEach((message) -> log.info(message));
+
+    // Wait for messages
+    assertTrue("Thread did not send all messages. Count: " + allMessagesSent.getCount(),
+        allMessagesSent.await(60, TimeUnit.SECONDS));
+  }
+
+  @Test
+  public void testQueueTimeout() throws InterruptedException {
+    System.setProperty("samza.container.name", "samza-container-1");
+
+    MockSystemProducerAppender systemProducerAppender = new MockSystemProducerAppender();
+    systemProducerAppender.queueTimeoutS = 1;
+    PatternLayout layout = new PatternLayout();
+    layout.setConversionPattern("%m");
+    systemProducerAppender.setLayout(layout);
+    systemProducerAppender.activateOptions();
+    log.addAppender(systemProducerAppender);
+
+    int extraMessageCount = 5;
+    int expectedMessagesSent = extraMessageCount - 1; // -1 because when the queue is drained there is one additional message that couldn't be added
+    List<String> messages = new ArrayList<>(StreamAppender.DEFAULT_QUEUE_SIZE + extraMessageCount);
+    for (int i = 0; i < StreamAppender.DEFAULT_QUEUE_SIZE + extraMessageCount; i++) {
+      messages.add(String.valueOf(i));
+    }
+
+    // Set up latch
+    final CountDownLatch allMessagesSent = new CountDownLatch(expectedMessagesSent); // We expect to drop all but the extra messages
+    final CountDownLatch waitForTimeout = new CountDownLatch(1);
+    MockSystemProducer.listeners.add((source, envelope) -> {
+        allMessagesSent.countDown();
+        try {
+          waitForTimeout.await();
+        } catch (InterruptedException e) {
+          fail("Test could not run properly because of a thread interrupt.");
+        }
+      });
+
+    // Log the messages. This is where the timeout will happen!
+    messages.forEach((message) -> log.info(message));
+
+    assertEquals(messages.size() - expectedMessagesSent, systemProducerAppender.metrics.logMessagesDropped.getCount());
+
+    // Allow all the rest of the messages to send.
+    waitForTimeout.countDown();
+
+    // Wait for messages
+    assertTrue("Thread did not send all messages. Count: " + allMessagesSent.getCount(),
+        allMessagesSent.await(60, TimeUnit.SECONDS));
+    assertEquals(expectedMessagesSent, MockSystemProducer.messagesReceived.size());
+  }
+
+  private void logAndVerifyMessages(List<String> messages) throws InterruptedException {
+    // Set up latch
+    final CountDownLatch allMessagesSent = new CountDownLatch(messages.size());
+    MockSystemProducer.listeners.add((source, envelope) -> allMessagesSent.countDown());
+
+    // Log the messages
+    messages.forEach((message) -> log.info(message));
+
+    // Wait for messages
+    assertTrue("Timeout while waiting for StreamAppender to send all messages. Count: " + allMessagesSent.getCount(),
+        allMessagesSent.await(60, TimeUnit.SECONDS));
+
+    // Verify
+    assertEquals(messages.size(), MockSystemProducer.messagesReceived.size());
+    for (int i = 0; i < messages.size(); i++) {
+      assertTrue("Message mismatch at index " + i,
+          new String((byte[]) MockSystemProducer.messagesReceived.get(i)).contains(asJsonMessageSegment(messages.get(i))));
+    }
+  }
+
+  private String asJsonMessageSegment(String message) {
+    return String.format("\"message\":\"%s\"", message);
   }
 
   /**

From ef1c9625c63b6c169585c5d1928298f36f3037fb Mon Sep 17 00:00:00 2001
From: Jacob Maes <jmakes@apache.org>
Date: Fri, 22 Dec 2017 10:36:37 -0800
Subject: [PATCH 20/36] =?UTF-8?q?SAMZA-1539:=20KafkaProducer=20potential?=
 =?UTF-8?q?=20hang=20on=20close()=20when=20task.drop.pr=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…oducer.errors==true

Author: Jacob Maes <jmakes@apache.org>

Reviewers: Boris Shkolnik <boryas@apache.org>

Closes #390 from jmakes/samza-1539
---
 .../system/kafka/KafkaSystemProducer.scala    | 135 ++++++++++--------
 .../kafka/TestKafkaSystemProducer.scala       |  39 +++--
 2 files changed, 103 insertions(+), 71 deletions(-)

diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemProducer.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemProducer.scala
index 5e8366612b..9eaf895052 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemProducer.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemProducer.scala
@@ -28,7 +28,6 @@ import org.apache.kafka.clients.producer.Producer
 import org.apache.kafka.clients.producer.ProducerRecord
 import org.apache.kafka.clients.producer.RecordMetadata
 import org.apache.kafka.common.PartitionInfo
-import org.apache.kafka.common.errors.SerializationException
 import org.apache.samza.system.OutgoingMessageEnvelope
 import org.apache.samza.system.SystemProducer
 import org.apache.samza.system.SystemProducerException
@@ -46,32 +45,30 @@ class KafkaSystemProducer(systemName: String,
 
   // Represents a fatal error that caused the producer to close.
   val fatalException: AtomicReference[SystemProducerException] = new AtomicReference[SystemProducerException]()
-  @volatile var producer: Producer[Array[Byte], Array[Byte]] = null
-  val producerLock: Object = new Object
+  val producerRef: AtomicReference[Producer[Array[Byte], Array[Byte]]] = new AtomicReference[Producer[Array[Byte], Array[Byte]]]()
+  val producerCreationLock: Object = new Object
+  @volatile var stopped = false
 
   def start(): Unit = {
-    producer = getProducer()
+    producerRef.set(getProducer())
   }
 
   def stop() {
     info("Stopping producer for system: " + this.systemName)
 
-    // stop() should not happen often so no need to optimize locking
-    producerLock.synchronized {
-      try {
-        if (producer != null) {
-          producer.close  // Also performs the equivalent of a flush()
-        }
+    stopped = true
+    val currentProducer = producerRef.getAndSet(null)
+    try {
+      if (currentProducer != null) {
+        currentProducer.close // Also performs the equivalent of a flush()
+      }
 
-        val exception = fatalException.get()
-        if (exception != null) {
-          error("Observed an earlier send() error while closing producer", exception)
-        }
-      } catch {
-        case e: Exception => error("Error while closing producer for system: " + systemName, e)
-      } finally {
-        producer = null
+      val exception = fatalException.get()
+      if (exception != null) {
+        error("Observed an earlier send() error while closing producer", exception)
       }
+    } catch {
+      case e: Exception => error("Error while closing producer for system: " + systemName, e)
     }
   }
 
@@ -82,7 +79,7 @@ class KafkaSystemProducer(systemName: String,
     trace("Enqueuing message: %s, %s." format (source, envelope))
 
     val topicName = envelope.getSystemStream.getStream
-    if (topicName == null || topicName == "") {
+    if (topicName == null || topicName.isEmpty) {
       throw new IllegalArgumentException("Invalid system stream: " + envelope.getSystemStream)
     }
 
@@ -92,10 +89,7 @@ class KafkaSystemProducer(systemName: String,
       throw new SystemProducerException("Producer was unable to recover from previous exception.", globalProducerException)
     }
 
-    val currentProducer = producer
-    if (currentProducer == null) {
-      throw new SystemProducerException("Kafka producer is null.")
-    }
+    val currentProducer = getOrCreateCurrentProducer
 
     // Java-based Kafka producer API requires an "Integer" type partitionKey and does not allow custom overriding of Partitioners
     // Any kind of custom partitioning has to be done on the client-side
@@ -115,7 +109,7 @@ class KafkaSystemProducer(systemName: String,
             val producerException = new SystemProducerException("Failed to send message for Source: %s on System:%s Topic:%s Partition:%s"
               .format(source, systemName, topicName, partitionKey), exception)
 
-            handleSendException(currentProducer, producerException, true)
+            handleFatalSendException(currentProducer, producerException)
           }
         }
       })
@@ -125,18 +119,25 @@ class KafkaSystemProducer(systemName: String,
         val producerException = new SystemProducerException("Failed to send message for Source: %s on System:%s Topic:%s Partition:%s"
           .format(source, systemName, topicName, partitionKey), originalException)
 
-        handleSendException(currentProducer, producerException, isFatalException(originalException))
+        metrics.sendFailed.inc
+        error("Got a synchronous error from Kafka producer.", producerException)
+        // Synchronous exceptions are always recoverable so propagate it up and let the user decide
         throw producerException
     }
   }
 
-
   def flush(source: String) {
     updateTimer(metrics.flushNs) {
       metrics.flushes.inc
 
-      val currentProducer = producer
+      val currentProducer = producerRef.get()
       if (currentProducer == null) {
+        if (dropProducerExceptions) {
+          // No producer to flush, but we're ignoring exceptions so just return.
+          warn("Skipping flush because the Kafka producer is null.")
+          metrics.flushFailed.inc
+          return
+        }
         throw new SystemProducerException("Kafka producer is null.")
       }
 
@@ -162,7 +163,14 @@ class KafkaSystemProducer(systemName: String,
   }
 
 
-  private def handleSendException(currentProducer: Producer[Array[Byte], Array[Byte]], producerException: SystemProducerException, isFatalException: Boolean) = {
+  /**
+    * Handles a fatal exception by closing the producer and either recreating it or storing the exception
+    * to rethrow later, depending on the value of dropProducerExceptions.
+    *
+    * @param currentProducer   the current producer for which the exception occurred. Must not be null.
+    * @param producerException the exception to handle.
+    */
+  private def handleFatalSendException(currentProducer: Producer[Array[Byte], Array[Byte]], producerException: SystemProducerException): Unit = {
     metrics.sendFailed.inc
     error(producerException)
     // The SystemProducer API is synchronous, so there's no way for us to guarantee that an exception will
@@ -172,49 +180,56 @@ class KafkaSystemProducer(systemName: String,
     if (dropProducerExceptions) {
       warn("Ignoring producer exception. All messages in the failed producer request will be dropped!")
 
-      if (isFatalException) {
-        producerLock.synchronized {
-          // Prevent each callback from recreating producer for the same failure.
-          if (currentProducer == producer) {
-            info("Creating a new producer for system %s." format systemName)
-            try {
-              currentProducer.close(0, TimeUnit.MILLISECONDS)
-            } catch {
-              case exception: Exception => error("Exception while closing producer.", exception)
-            }
-            producer = getProducer()
-          }
-        }
-      }
-    } else {
-      // If there is an exception in the callback, it means that the Kafka producer has exhausted the max-retries
-      // Close producer to ensure messages queued in-flight are not sent and hence, avoid re-ordering
-      // This works because there is only 1 callback thread and no sends can complete until the callback returns.
-      if (isFatalException) {
-        fatalException.compareAndSet(null, producerException)
+      // Prevent each callback from closing and nulling producer for the same failure.
+      if (currentProducer == producerRef.get()) {
+        info("Closing producer for system %s." format systemName)
         try {
+          // send()s can get ProducerClosedException if the producer is stopped after they get the currentProducer
+          // reference but before producer.send() returns. That's ONLY ok when dropProducerExceptions is true.
+          // Also, when producer.close(0) is invoked on the Kafka IO thread, when it returns there will be no more
+          // messages sent over the wire. This is key to ensuring no out-of-order messages as a result of recreating
+          // the producer.
           currentProducer.close(0, TimeUnit.MILLISECONDS)
         } catch {
           case exception: Exception => error("Exception while closing producer.", exception)
         }
+        producerRef.compareAndSet(currentProducer, null)
+      }
+    } else {
+      // If there is an exception in the callback, it means that the Kafka producer has exhausted the max-retries
+      // Close producer to ensure messages queued in-flight are not sent and hence, avoid re-ordering
+      // This works because there is only 1 IO thread and no IO can be done until the callback returns.
+      // Do not create a new producer here! It cannot be done without data loss for all concurrency modes.
+      fatalException.compareAndSet(null, producerException)
+      try {
+        currentProducer.close(0, TimeUnit.MILLISECONDS)
+      } catch {
+        case exception: Exception => error("Exception while closing producer.", exception)
       }
     }
   }
 
   /**
-    * A fatal exception is one that corrupts the producer or otherwise makes it unusable.
-    * We want to handle non-fatal exceptions differently because they can often be handled by the user
-    * and that's preferable because it gives users that drop exceptions a way to do that with less
-    * data loss (no collateral damage from batches of messages getting dropped)
-    *
-    * @param exception  the exception to check
-    * @return           true if the exception is unrecoverable.
+    * @return the current producer. Never returns null.
     */
-  private def isFatalException(exception: Exception): Boolean = {
-    exception match {
-      case _: SerializationException => false
-      case _: ClassCastException => false
-      case _ => true
+  private def getOrCreateCurrentProducer = {
+    var currentProducer = producerRef.get
+
+    if (currentProducer == null) {
+      if (dropProducerExceptions && !stopped) {
+        // Note: While this lock prevents others from creating a new producer, they could still set it to null.
+        producerCreationLock.synchronized {
+          currentProducer = producerRef.get
+          if (currentProducer == null) {
+            currentProducer = getProducer()
+            producerRef.set(currentProducer)
+          }
+        }
+        // Invariant: currentProducer must not be null at this point.
+      } else {
+        throw new SystemProducerException("Kafka producer is null.")
+      }
     }
+    currentProducer
   }
 }
\ No newline at end of file
diff --git a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemProducer.scala b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemProducer.scala
index 9117be5380..d874a07d3a 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemProducer.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemProducer.scala
@@ -41,7 +41,7 @@ class TestKafkaSystemProducer {
     systemProducer.register("test")
     systemProducer.start
     systemProducer.send("test", someMessage)
-    assertEquals(1, systemProducer.producer.asInstanceOf[MockProducer[Array[Byte], Array[Byte]]].history().size())
+    assertEquals(1, systemProducer.producerRef.get().asInstanceOf[MockProducer[Array[Byte], Array[Byte]]].history().size())
     systemProducer.stop
   }
 
@@ -207,7 +207,7 @@ class TestKafkaSystemProducer {
     val mockProducer = new MockKafkaProducer(1, "test", 1)
     val producer = new KafkaSystemProducer(systemName =  "test",
                                            getProducer = () => {
-                                             mockProducer.open() // A new producer is never closed
+                                             mockProducer.open() // A new producer would not already be closed, so reset it.
                                              mockProducer
                                            },
                                            metrics = producerMetrics)
@@ -219,6 +219,7 @@ class TestKafkaSystemProducer {
     mockProducer.setErrorNext(true, true, new RecordTooLargeException())
     producer.send("test", msg3) // Callback exception
     assertTrue(mockProducer.isClosed)
+    assertNotNull(producer.producerRef.get())
     assertEquals("Should NOT have created a new producer", 1, mockProducer.getOpenCount)
 
     val senderException = intercept[SystemProducerException] {
@@ -269,7 +270,7 @@ class TestKafkaSystemProducer {
     val mockProducer = new MockKafkaProducer(1, "test", 1)
     val producer = new KafkaSystemProducer(systemName =  "test",
       getProducer = () => {
-        mockProducer.open() // A new producer is never closed
+        mockProducer.open() // A new producer would not already be closed, so reset it.
         mockProducer
       },
       metrics = producerMetrics)
@@ -286,6 +287,7 @@ class TestKafkaSystemProducer {
     mockProducer.setErrorNext(true, true, new RecordTooLargeException())
     producer.send("test1", msg3) // Callback exception
     assertTrue(mockProducer.isClosed)
+    assertNotNull(producer.producerRef.get())
     assertEquals("Should NOT have created a new producer", 1, mockProducer.getOpenCount)
 
     // Subsequent sends
@@ -343,7 +345,7 @@ class TestKafkaSystemProducer {
     val mockProducer = new MockKafkaProducer(1, "test", 1)
     val producer = new KafkaSystemProducer(systemName = "test",
       getProducer = () => {
-        mockProducer.open() // A new producer is never closed
+        mockProducer.open() // A new producer would not already be closed, so reset it.
         mockProducer
       },
       metrics = producerMetrics)
@@ -359,6 +361,7 @@ class TestKafkaSystemProducer {
     }
     assertTrue(sendException.getCause.isInstanceOf[SerializationException])
     assertFalse(mockProducer.isClosed)
+    assertNotNull(producer.producerRef.get())
     assertEquals("Should NOT have created a new producer", 1, mockProducer.getOpenCount)
 
     producer.send("test1", msg3) // Should be able to resend msg3
@@ -406,7 +409,7 @@ class TestKafkaSystemProducer {
     val mockProducer = new MockKafkaProducer(1, "test", 1)
     val producer = new KafkaSystemProducer(systemName =  "test",
       getProducer = () => {
-        mockProducer.open() // A new producer is never closed
+        mockProducer.open() // A new producer would not already be closed, so reset it.
         mockProducer
       },
       metrics = producerMetrics,
@@ -418,13 +421,18 @@ class TestKafkaSystemProducer {
     producer.send("test", msg2)
     mockProducer.setErrorNext(true, true, new RecordTooLargeException())
     producer.send("test", msg3) // Callback exception
-    assertFalse(mockProducer.isClosed)
-    assertEquals("Should have created a new producer", 2, mockProducer.getOpenCount)
+    assertTrue(mockProducer.isClosed)
+    assertNull(producer.producerRef.get())
+    assertEquals("Should not have created a new producer", 1, mockProducer.getOpenCount)
 
     producer.send("test", msg4) // Should succeed because the producer recovered.
+    assertFalse(mockProducer.isClosed)
+    assertNotNull(producer.producerRef.get())
+    assertEquals("Should have created a new producer", 2, mockProducer.getOpenCount)
     producer.flush("test") // Should not throw
 
     producer.send("test", msg5) // Should be able to send again after flush
+    assertEquals("Should not have created a new producer", 2, mockProducer.getOpenCount)
     producer.flush("test")
 
     assertEquals(4, mockProducer.getMsgsSent) // every message except the one with the error should get sent
@@ -456,7 +464,7 @@ class TestKafkaSystemProducer {
     val mockProducer = new MockKafkaProducer(1, "test", 1)
     val producer = new KafkaSystemProducer(systemName =  "test",
       getProducer = () => {
-        mockProducer.open() // A new producer is never closed
+        mockProducer.open() // A new producer would not already be closed, so reset it.
         mockProducer
       },
       metrics = producerMetrics,
@@ -473,12 +481,17 @@ class TestKafkaSystemProducer {
     // Inject error for next send
     mockProducer.setErrorNext(true, true, new RecordTooLargeException())
     producer.send("test1", msg3) // Callback exception
-    assertFalse(mockProducer.isClosed)
-    assertEquals("Should have created a new producer", 2, mockProducer.getOpenCount)
+    assertTrue(mockProducer.isClosed)
+    assertNull(producer.producerRef.get())
+    assertEquals("Should not have created a new producer", 1, mockProducer.getOpenCount)
 
     // Subsequent sends
     producer.send("test1", msg4) // Should succeed because the producer recovered.
+    assertFalse(mockProducer.isClosed)
+    assertEquals("Should have created a new producer", 2, mockProducer.getOpenCount)
+    assertNotNull(producer.producerRef.get())
     producer.send("test2", msg5) // Second source should also not have any error.
+    assertEquals("Should not have created a new producer", 2, mockProducer.getOpenCount)
 
     // Flushes
     producer.flush("test2") // Should not throw for test2
@@ -503,7 +516,7 @@ class TestKafkaSystemProducer {
     val mockProducer = new MockKafkaProducer(1, "test", 1)
     val producer = new KafkaSystemProducer(systemName = "test",
       getProducer = () => {
-        mockProducer.open() // A new producer is never closed
+        mockProducer.open() // A new producer would not already be closed, so reset it.
         mockProducer
       },
       metrics = producerMetrics,
@@ -520,9 +533,13 @@ class TestKafkaSystemProducer {
     }
     assertTrue(sendException.getCause.isInstanceOf[SerializationException])
     assertFalse(mockProducer.isClosed)
+    assertNotNull(producer.producerRef.get()) // Synchronous error; producer should not be recreated
     assertEquals("Should NOT have created a new producer", 1, mockProducer.getOpenCount)
 
     producer.send("test1", msg3) // Should be able to resend msg3
+    assertFalse(mockProducer.isClosed)
+    assertEquals("Should NOT have created a new producer", 1, mockProducer.getOpenCount)
+    assertNotNull(producer.producerRef.get())
     producer.send("test2", msg4) // Second source should not be affected
 
     producer.flush("test1") // Flush should be unaffected

From 9a8099cdaa2a8ee6fa64d263ebf6704ee5a108da Mon Sep 17 00:00:00 2001
From: Jacob Maes <jmakes@apache.org>
Date: Fri, 22 Dec 2017 11:41:27 -0800
Subject: [PATCH 21/36] Fix a couple sonarcloud issues with samza-1537

Author: Jacob Maes <jmakes@apache.org>

Reviewers: Jagadish <jvenkatr@linkedin.com>

Closes #393 from jmakes/streamappender-sonarcloud
---
 .../java/org/apache/samza/logging/log4j/StreamAppender.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java b/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java
index 0ea8b681c9..5f41959df7 100644
--- a/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java
+++ b/samza-log4j/src/main/java/org/apache/samza/logging/log4j/StreamAppender.java
@@ -152,7 +152,7 @@ public void append(LoggingEvent event) {
             // Emit a metric which can be monitored to ensure it doesn't happen often.
             metrics.logMessagesDropped.inc(messagesDropped);
           }
-          metrics.bufferFillPct.set(Math.round(100 * logQueue.size() / DEFAULT_QUEUE_SIZE));
+          metrics.bufferFillPct.set(Math.round(100f * logQueue.size() / DEFAULT_QUEUE_SIZE));
         }
       } catch (Exception e) {
         System.err.println("[StreamAppender] Error sending log message:");
@@ -188,7 +188,8 @@ public void close() {
       try {
         transferThread.join();
       } catch (InterruptedException e) {
-        log.error("Interrupted while waiting for sink thread to finish.", e);
+        log.error("Interrupted while waiting for transfer thread to finish.", e);
+        Thread.currentThread().interrupt();
       }
 
       flushSystemProducer();

From 93219c78d6f473d2c7ad4d8702a25f704ee61a7a Mon Sep 17 00:00:00 2001
From: Jagadish <jvenkatraman@linkedin.com>
Date: Fri, 22 Dec 2017 13:58:33 -0800
Subject: [PATCH 22/36] SAMZA-1547; Parse default value grouper-factory config
 in KafkaCheckpointMgr

- Additionally, updated all unit-tests.

Author: Jagadish <jvenkatraman@linkedin.com>

Reviewers: Prateek M <prmaheshw@linkedin.com>

Closes #394 from vjagadish1989/kcm-fix
---
 .../apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala  | 2 +-
 .../samza/checkpoint/kafka/TestKafkaCheckpointManager.scala     | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala b/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala
index 217b2b6b10..e1187c5ce5 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala
@@ -59,7 +59,7 @@ class KafkaCheckpointManager(checkpointSpec: KafkaStreamSpec,
   val checkpointTopic: String = checkpointSpec.getPhysicalName
   val checkpointSsp = new SystemStreamPartition(checkpointSystem, checkpointTopic, new Partition(0))
   val checkpointKeySerde = new KafkaCheckpointLogKeySerde
-  val expectedGrouperFactory = config.get(JobConfig.SSP_GROUPER_FACTORY)
+  val expectedGrouperFactory = new JobConfig(config).getSystemStreamPartitionGrouperFactory
 
   val systemProducer = systemFactory.getProducer(checkpointSystem, config, metricsRegistry)
   val systemConsumer = systemFactory.getConsumer(checkpointSystem, config, metricsRegistry)
diff --git a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala
index dcf4068517..ec9f3a0952 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala
@@ -150,7 +150,6 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness {
     new MapConfig(new ImmutableMap.Builder[String, String]()
       .put(JobConfig.JOB_NAME, "some-job-name")
       .put(JobConfig.JOB_ID, "i001")
-      .put(JobConfig.SSP_GROUPER_FACTORY, sspGrouperFactoryName)
       .put(s"systems.$checkpointSystemName.samza.factory", classOf[KafkaSystemFactory].getCanonicalName)
       .put(s"systems.$checkpointSystemName.producer.bootstrap.servers", brokers)
       .put(s"systems.$checkpointSystemName.consumer.zookeeper.connect", zkConnect)

From 882f61a6925d3950e408ec8e202c0e8b6cdff450 Mon Sep 17 00:00:00 2001
From: Jagadish <jvenkatraman@linkedin.com>
Date: Tue, 2 Jan 2018 12:16:51 -0800
Subject: [PATCH 23/36] SAMZA-1528: Change ClusterResourceManager to use the
 async NMClient

- Rewrite container handling to be asynchronous
- Verified various failure scenarios using Unit tests, and deployments of a local Samza job.

Author: Jagadish <jvenkatraman@linkedin.com>
Author: Fred Ji <haifeng.ji@gmail.com>
Author: Srinivasulu Punuru <spunuru@linkedin.com>

Reviewers: Jacob Maes<jmakes@linkedin.com>, Xinyu Liu<xinyuiscool@gmail.com>

Closes #380 from vjagadish1989/cluster-mgr-refactor1
---
 .../AbstractContainerAllocator.java           |  17 +-
 .../ClusterResourceManager.java               |  26 +-
 .../ContainerProcessManager.java              |  74 ++++-
 .../clustermanager/SamzaApplicationState.java |   8 +-
 .../samza/clustermanager/SamzaResource.java   |   8 +
 .../samza/operators/impl/OperatorImpl.java    |  16 +-
 .../MockClusterResourceManager.java           |  19 +-
 .../MockClusterResourceManagerCallback.java   |  10 +
 .../TestContainerAllocator.java               |  55 ----
 .../TestContainerProcessManager.java          | 101 ++++--
 .../TestHostAwareContainerAllocator.java      |  56 ----
 .../operators/impl/TestOperatorImpl.java      |   1 +
 .../apache/samza/job/yarn/YarnAppState.java   |   4 +-
 .../job/yarn/YarnClusterResourceManager.java  | 310 ++++++++++++++++--
 .../samza/job/yarn/YarnContainerRunner.java   | 272 ---------------
 15 files changed, 498 insertions(+), 479 deletions(-)
 delete mode 100644 samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnContainerRunner.java

diff --git a/samza-core/src/main/java/org/apache/samza/clustermanager/AbstractContainerAllocator.java b/samza-core/src/main/java/org/apache/samza/clustermanager/AbstractContainerAllocator.java
index b83d83c017..1b7085752d 100644
--- a/samza-core/src/main/java/org/apache/samza/clustermanager/AbstractContainerAllocator.java
+++ b/samza-core/src/main/java/org/apache/samza/clustermanager/AbstractContainerAllocator.java
@@ -151,20 +151,11 @@ protected void runStreamProcessor(SamzaResourceRequest request, String preferred
     log.info("Found available resources on {}. Assigning request for container_id {} with "
             + "timestamp {} to resource {}",
         new Object[]{preferredHost, String.valueOf(containerID), request.getRequestTimestampMs(), resource.getResourceID()});
-    try {
-      //launches a StreamProcessor on the resource
-      clusterResourceManager.launchStreamProcessor(resource, builder);
 
-      if (state.neededContainers.decrementAndGet() == 0) {
-        state.jobHealthy.set(true);
-      }
-      state.runningContainers.put(request.getContainerID(), resource);
-
-    } catch (SamzaContainerLaunchException e) {
-      log.warn(String.format("Got exception while starting resource %s. Requesting a new resource on any host", resource), e);
-      resourceRequestState.releaseUnstartableContainer(resource);
-      requestResource(containerID, ResourceRequestState.ANY_HOST);
-    }
+    //Submit a request to launch a StreamProcessor on the provided resource. To match with the response returned later
+    //in the callback, we should also store state about the container whose launch is pending.
+    clusterResourceManager.launchStreamProcessor(resource, builder);
+    state.pendingContainers.put(containerID, resource);
   }
 
   /**
diff --git a/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterResourceManager.java b/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterResourceManager.java
index 715cf6675d..f8a8c8b377 100644
--- a/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterResourceManager.java
+++ b/samza-core/src/main/java/org/apache/samza/clustermanager/ClusterResourceManager.java
@@ -105,15 +105,21 @@ public ClusterResourceManager(Callback callback) {
   public abstract void releaseResources(SamzaResource resource);
 
   /***
-   * Requests the launch of a StreamProcessor with the specified context on the resource.
+   * Requests the launch of a StreamProcessor with the specified context on the resource asynchronously.
+   *
+   * <p>
+   *   Either {@link Callback#onStreamProcessorLaunchSuccess(SamzaResource)} or
+   *   {@link Callback#onStreamProcessorLaunchFailure(SamzaResource, Throwable)} will be invoked
+   *   to indicate the result of this operation.
+   * </p>
+   *
    * @param resource the specified resource
    * @param builder A builder implementation that encapsulates the context for the
    *                StreamProcessor. A builder encapsulates the ID for the processor, the
    *                build environment, the command to execute etc.
-   * @throws SamzaContainerLaunchException  when there's an error during the requesting launch.
    *
    */
-  public abstract void launchStreamProcessor(SamzaResource resource, CommandBuilder builder) throws SamzaContainerLaunchException;
+  public abstract void launchStreamProcessor(SamzaResource resource, CommandBuilder builder);
 
 
   public abstract void stop(SamzaApplicationState.SamzaAppStatus status);
@@ -143,6 +149,20 @@ public interface Callback {
      */
     void onResourcesCompleted(List<SamzaResourceStatus> resources);
 
+
+    /**
+     * Callback invoked when the launch request for a StreamProcessor on the {@link SamzaResource} is successful.
+     * @param resource the resource on which the StreamProcessor is launched
+     */
+    void onStreamProcessorLaunchSuccess(SamzaResource resource);
+
+    /**
+     * Callback invoked when there is a failure in launching a StreamProcessor on the provided {@link SamzaResource}.
+     * @param resource the resource on which the StreamProcessor was submitted for launching
+     * @param t the error in launching the StreamProcessor
+     */
+    void onStreamProcessorLaunchFailure(SamzaResource resource, Throwable t);
+
     /***
      * This callback is invoked when there is an error in the ClusterResourceManager. This is
      * guaranteed to be invoked when there is an uncaught exception in any other
diff --git a/samza-core/src/main/java/org/apache/samza/clustermanager/ContainerProcessManager.java b/samza-core/src/main/java/org/apache/samza/clustermanager/ContainerProcessManager.java
index 6a18b8437a..474ac8c433 100644
--- a/samza-core/src/main/java/org/apache/samza/clustermanager/ContainerProcessManager.java
+++ b/samza-core/src/main/java/org/apache/samza/clustermanager/ContainerProcessManager.java
@@ -105,6 +105,18 @@ public class ContainerProcessManager implements ClusterResourceManager.Callback
    */
   private final ContainerProcessManagerMetrics metrics;
 
+  //for testing
+  ContainerProcessManager(Config config, SamzaApplicationState state, MetricsRegistryMap registry, AbstractContainerAllocator allocator, ClusterResourceManager manager) {
+    this.state = state;
+    this.clusterManagerConfig = new ClusterManagerConfig(config);
+    this.jobConfig = new JobConfig(config);
+    this.hostAffinityEnabled = clusterManagerConfig.getHostAffinityEnabled();
+    this.clusterResourceManager = manager;
+    this.metrics = new ContainerProcessManagerMetrics(config, state, registry);
+    this.containerAllocator = allocator;
+    this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread");
+  }
+
   public ContainerProcessManager(Config config,
                                  SamzaApplicationState state,
                                  MetricsRegistryMap registry) {
@@ -153,7 +165,6 @@ public ContainerProcessManager(Config config,
 
     this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread");
     log.info("finished initialization of samza task manager");
-
   }
 
   public boolean shouldShutdown() {
@@ -383,6 +394,50 @@ public void onResourcesCompleted(List<SamzaResourceStatus> resourceStatuses) {
     }
   }
 
+  @Override
+  public void onStreamProcessorLaunchSuccess(SamzaResource resource) {
+
+    // 1. Obtain the Samza container Id for the pending container on this resource.
+    String containerId = getPendingContainerId(resource.getResourceID());
+    log.info("Successfully started container ID: {} on resource: {}", containerId, resource);
+
+    // 2. Remove the container from the pending buffer and add it to the running buffer. Additionally, update the
+    // job-health metric.
+    if (containerId != null) {
+      log.info("Moving containerID: {} on resource: {} from pending to running state", containerId, resource);
+      state.pendingContainers.remove(containerId);
+      state.runningContainers.put(containerId, resource);
+
+      if (state.neededContainers.decrementAndGet() == 0) {
+        state.jobHealthy.set(true);
+      }
+    } else {
+      log.warn("SamzaResource {} was not in pending state. Got an invalid callback for a launch request that " +
+          "was not issued", resource);
+    }
+  }
+
+  @Override
+  public void onStreamProcessorLaunchFailure(SamzaResource resource, Throwable t) {
+    log.error("Got a launch failure for SamzaResource {} with exception {}", resource, t);
+    // 1. Release resources for containers that failed back to YARN
+    log.info("Releasing unstartable container {}", resource.getResourceID());
+    clusterResourceManager.releaseResources(resource);
+
+    // 2. Obtain the Samza container Id for the pending container on this resource.
+    String containerId = getPendingContainerId(resource.getResourceID());
+    log.info("Failed container ID: {} for resourceId: {}", containerId, resource.getResourceID());
+
+    // 3. Re-request resources on ANY_HOST in case of launch failures on the preferred host.
+    if (containerId != null) {
+      log.info("Launch of container ID: {} failed on host: {}. Falling back to ANY_HOST", containerId, resource.getHost());
+      containerAllocator.requestResource(containerId, ResourceRequestState.ANY_HOST);
+    } else {
+      log.warn("SamzaResource {} was not in pending state. Got an invalid callback for a launch request that was " +
+          "not issued", resource);
+    }
+  }
+
   /**
    * An error in the callback terminates the JobCoordinator
    * @param e the underlying exception/error
@@ -419,6 +474,21 @@ private ResourceManagerFactory getContainerProcessManagerFactory(final ClusterMa
     return factory;
   }
 
+  /**
+   * Obtains the ID of the Samza container pending launch on the provided resource.
+   *
+   * @param resourceId the Id of the resource
+   * @return the Id of the Samza container on this resource
+   */
+  private String getPendingContainerId(String resourceId) {
+    for (Map.Entry<String, SamzaResource> entry: state.pendingContainers.entrySet()) {
+      if (entry.getValue().getResourceID().equals(resourceId)) {
+        log.info("Matching container ID found " + entry.getKey() + " " + entry.getValue());
+        return entry.getKey();
+      }
+    }
+    return null;
+  }
 
 
-}
+}
\ No newline at end of file
diff --git a/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaApplicationState.java b/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaApplicationState.java
index adc6e5159e..0dcaace9c0 100644
--- a/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaApplicationState.java
+++ b/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaApplicationState.java
@@ -99,7 +99,13 @@ public enum SamzaAppStatus { UNDEFINED, SUCCEEDED, FAILED }
   public final AtomicInteger neededContainers = new AtomicInteger(0);
 
   /**
-   *  Map of the samzaContainerId to the {@link SamzaResource} on which it is running
+   *  Map of the samzaContainerId to the {@link SamzaResource} on which it is submitted for launch.
+   *  Modified by both the NMCallback and the ContainerAllocator thread.
+   */
+  public final ConcurrentMap<String, SamzaResource> pendingContainers = new ConcurrentHashMap<String, SamzaResource>(0);
+
+  /**
+   *  Map of the samzaContainerId to the {@link SamzaResource} on which it is running.
    *  Modified by both the AMRMCallbackThread and the ContainerAllocator thread
    */
   public final ConcurrentMap<String, SamzaResource> runningContainers = new ConcurrentHashMap<String, SamzaResource>(0);
diff --git a/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaResource.java b/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaResource.java
index ba6ca2cb6c..2927491d57 100644
--- a/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaResource.java
+++ b/samza-core/src/main/java/org/apache/samza/clustermanager/SamzaResource.java
@@ -60,6 +60,14 @@ public int hashCode() {
     return result;
   }
 
+  @Override
+  public String toString() {
+    return "SamzaResource{" +
+        "host='" + host + '\'' +
+        ", resourceID='" + resourceID + '\'' +
+        '}';
+  }
+
   public int getNumCores() {
     return numCores;
   }
diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImpl.java b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImpl.java
index 862e5f9991..9b2b4cf1f2 100644
--- a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImpl.java
+++ b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImpl.java
@@ -63,7 +63,7 @@ public abstract class OperatorImpl<M, RM> {
   private Counter numMessage;
   private Timer handleMessageNs;
   private Timer handleTimerNs;
-  private long inputWatermark = WatermarkStates.WATERMARK_NOT_EXIST;
+  private long currentWatermark = WatermarkStates.WATERMARK_NOT_EXIST;
   private long outputWatermark = WatermarkStates.WATERMARK_NOT_EXIST;
   private TaskName taskName;
   // Although the operator node is in the operator graph, the current task may not consume any message in it.
@@ -340,22 +340,22 @@ private final void onWatermark(long watermark, MessageCollector collector, TaskC
       inputWatermarkMin = prevOperators.stream().map(op -> op.getOutputWatermark()).min(Long::compare).get();
     }
 
-    if (inputWatermark < inputWatermarkMin) {
+    if (currentWatermark < inputWatermarkMin) {
       // advance the watermark time of this operator
-      inputWatermark = inputWatermarkMin;
-      LOG.trace("Advance input watermark to {} in operator {}", inputWatermark, getOpImplId());
+      currentWatermark = inputWatermarkMin;
+      LOG.trace("Advance input watermark to {} in operator {}", currentWatermark, getOpImplId());
 
       final Long outputWm;
       final Collection<RM> output;
       final WatermarkFunction watermarkFn = getOperatorSpec().getWatermarkFn();
       if (watermarkFn != null) {
         // user-overrided watermark handling here
-        output = (Collection<RM>) watermarkFn.processWatermark(inputWatermark);
+        output = (Collection<RM>) watermarkFn.processWatermark(currentWatermark);
         outputWm = watermarkFn.getOutputWatermark();
       } else {
         // use samza-provided watermark handling
         // default is to propagate the input watermark
-        output = handleWatermark(inputWatermark, collector, coordinator);
+        output = handleWatermark(currentWatermark, collector, coordinator);
         outputWm = getOutputWatermark();
       }
 
@@ -398,7 +398,7 @@ protected Collection<RM> handleWatermark(long inputWatermark, MessageCollector c
 
   /* package private for testing */
   final long getInputWatermark() {
-    return this.inputWatermark;
+    return this.currentWatermark;
   }
 
   /**
@@ -409,7 +409,7 @@ final long getInputWatermark() {
   protected long getOutputWatermark() {
     if (usedInCurrentTask) {
       // default as input
-      return getInputWatermark();
+      return this.currentWatermark;
     } else {
       // always emit the max to indicate no input will be emitted afterwards
       return Long.MAX_VALUE;
diff --git a/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManager.java b/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManager.java
index 0d13fb1d4e..452cadcefb 100644
--- a/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManager.java
+++ b/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManager.java
@@ -19,10 +19,10 @@
 
 package org.apache.samza.clustermanager;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.samza.job.CommandBuilder;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -47,9 +47,11 @@ public void start() {
 
   @Override
   public void requestResources(SamzaResourceRequest resourceRequest) {
-    SamzaResource resource = new SamzaResource(resourceRequest.getNumCores(), resourceRequest.getMemoryMB(), resourceRequest.getPreferredHost(), UUID.randomUUID().toString());
-    List<SamzaResource> resources = Collections.singletonList(resource);
-    resourceRequests.addAll(resources);
+    SamzaResource resource = new SamzaResource(resourceRequest.getNumCores(), resourceRequest.getMemoryMB(),
+        resourceRequest.getPreferredHost(), UUID.randomUUID().toString());
+    resourceRequests.add(resource);
+
+    clusterManagerCallback.onResourcesAvailable(ImmutableList.of(resource));
   }
 
   @Override
@@ -63,15 +65,16 @@ public void releaseResources(SamzaResource resource) {
   }
 
   @Override
-  public void launchStreamProcessor(SamzaResource resource, CommandBuilder builder) throws SamzaContainerLaunchException {
+  public void launchStreamProcessor(SamzaResource resource, CommandBuilder builder)  {
     if (nextException != null) {
-      throw new SamzaContainerLaunchException(nextException);
+      clusterManagerCallback.onStreamProcessorLaunchFailure(resource, new SamzaContainerLaunchException(nextException));
+    } else {
+      launchedResources.add(resource);
+      clusterManagerCallback.onStreamProcessorLaunchSuccess(resource);
     }
-    launchedResources.add(resource);
     for (MockContainerListener listener : mockContainerListeners) {
       listener.postRunContainer(launchedResources.size());
     }
-
   }
 
   @Override
diff --git a/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManagerCallback.java b/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManagerCallback.java
index 507962506a..4e6e2c94a1 100644
--- a/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManagerCallback.java
+++ b/samza-core/src/test/java/org/apache/samza/clustermanager/MockClusterResourceManagerCallback.java
@@ -37,6 +37,16 @@ public void onResourcesCompleted(List<SamzaResourceStatus> resourceStatusList) {
     resourceStatuses.addAll(resourceStatusList);
   }
 
+  @Override
+  public void onStreamProcessorLaunchSuccess(SamzaResource resource) {
+    // no op
+  }
+
+  @Override
+  public void onStreamProcessorLaunchFailure(SamzaResource resource, Throwable t) {
+    // no op
+  }
+
   @Override
   public void onError(Throwable e) {
     error = e;
diff --git a/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerAllocator.java b/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerAllocator.java
index 734043a39b..459667309f 100644
--- a/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerAllocator.java
+++ b/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerAllocator.java
@@ -30,7 +30,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.HashMap;
 import java.util.Map;
@@ -202,58 +201,4 @@ public void run() {
     listener.verify();
   }
 
-
-  /**
-   * If the container fails to start e.g because it fails to connect to a NM on a host that
-   * is down, the allocator should request a new container on a different host.
-   */
-  @Test
-  public void testRerequestOnAnyHostIfContainerStartFails() throws Exception {
-    final SamzaResource container = new SamzaResource(1, 1024, "2", "id0");
-    final SamzaResource container1 = new SamzaResource(1, 1024, "2", "id0");
-    manager.nextException = new IOException("Cant connect to RM");
-
-    // Set up our final asserts before starting the allocator thread
-    MockContainerListener listener = new MockContainerListener(2, 1, 2, 0, null, new Runnable() {
-      @Override
-      public void run() {
-        // The failed container should be released. The successful one should not.
-        assertNotNull(manager.releasedResources);
-        assertEquals(1, manager.releasedResources.size());
-        assertTrue(manager.releasedResources.contains(container));
-      }
-    },
-        new Runnable() {
-          @Override
-          public void run() {
-            // Test that the first request assignment had a preferred host and the retry didn't
-            assertEquals(2, requestState.assignedRequests.size());
-
-            SamzaResourceRequest request = requestState.assignedRequests.remove();
-            assertEquals("0", request.getContainerID());
-            assertEquals("2", request.getPreferredHost());
-
-            request = requestState.assignedRequests.remove();
-            assertEquals("0", request.getContainerID());
-            assertEquals("ANY_HOST", request.getPreferredHost());
-
-            // This routine should be called after the retry is assigned, but before it's started.
-            // So there should still be 1 container needed.
-            assertEquals(1, state.neededContainers.get());
-          }
-        }, null
-    );
-    state.neededContainers.set(1);
-    requestState.registerContainerListener(listener);
-
-    containerAllocator.requestResource("0", "2");
-    containerAllocator.addResource(container);
-    containerAllocator.addResource(container1);
-    allocatorThread.start();
-
-    listener.verify();
-
-  }
-
-
 }
diff --git a/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerProcessManager.java b/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerProcessManager.java
index e252b7d7cd..5c2fe4a3d4 100644
--- a/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerProcessManager.java
+++ b/samza-core/src/test/java/org/apache/samza/clustermanager/TestContainerProcessManager.java
@@ -48,7 +48,7 @@
 
 public class TestContainerProcessManager {
   private final MockClusterResourceManagerCallback callback = new MockClusterResourceManagerCallback();
-  private final MockClusterResourceManager manager = new MockClusterResourceManager(callback);
+  private final MockClusterResourceManager clusterResourceManager = new MockClusterResourceManager(callback);
 
   private static volatile boolean isRunning = false;
 
@@ -125,7 +125,7 @@ public void testContainerProcessManager() throws Exception {
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
 
     AbstractContainerAllocator allocator =
@@ -145,7 +145,7 @@ public void testContainerProcessManager() throws Exception {
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
 
     allocator =
@@ -165,11 +165,11 @@ public void testOnInit() throws Exception {
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
 
     MockContainerAllocator allocator = new MockContainerAllocator(
-        manager,
+        clusterResourceManager,
         conf,
         state);
 
@@ -207,7 +207,7 @@ public void testOnShutdown() throws Exception {
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
     taskManager.start();
 
@@ -231,11 +231,11 @@ public void testTaskManagerShouldStopWhenContainersFinish() throws Exception {
             new MapConfig(conf),
             state,
             new MetricsRegistryMap(),
-            manager
+        clusterResourceManager
     );
 
     MockContainerAllocator allocator = new MockContainerAllocator(
-            manager,
+        clusterResourceManager,
             conf,
             state);
 
@@ -258,6 +258,7 @@ public void testTaskManagerShouldStopWhenContainersFinish() throws Exception {
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
+    taskManager.onStreamProcessorLaunchSuccess(container);
     assertFalse(taskManager.shouldShutdown());
 
     taskManager.onResourceCompleted(new SamzaResourceStatus("id0", "diagnostics", SamzaResourceStatus.SUCCESS));
@@ -278,11 +279,11 @@ public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
 
     MockContainerAllocator allocator = new MockContainerAllocator(
-        manager,
+        clusterResourceManager,
         conf,
         state);
 
@@ -305,7 +306,7 @@ public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
-
+    taskManager.onStreamProcessorLaunchSuccess(container);
     // Create first container failure
     taskManager.onResourceCompleted(new SamzaResourceStatus(container.getResourceID(), "diagnostics", 1));
 
@@ -316,8 +317,8 @@ public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception
 
     assertFalse(taskManager.shouldShutdown());
     assertFalse(state.jobHealthy.get());
-    assertEquals(2, manager.resourceRequests.size());
-    assertEquals(0, manager.releasedResources.size());
+    assertEquals(2, clusterResourceManager.resourceRequests.size());
+    assertEquals(0, clusterResourceManager.releasedResources.size());
 
     taskManager.onResourceAllocated(container);
 
@@ -325,7 +326,7 @@ public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
-
+    taskManager.onStreamProcessorLaunchSuccess(container);
     assertTrue(state.jobHealthy.get());
 
     // Create a second failure
@@ -334,8 +335,8 @@ public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception
 
     // The above failure should trigger a job shutdown because our retry count is set to 1
     assertEquals(0, allocator.getContainerRequestState().numPendingRequests());
-    assertEquals(2, manager.resourceRequests.size());
-    assertEquals(0, manager.releasedResources.size());
+    assertEquals(2, clusterResourceManager.resourceRequests.size());
+    assertEquals(0, clusterResourceManager.releasedResources.size());
     assertFalse(state.jobHealthy.get());
     assertTrue(taskManager.shouldShutdown());
     assertEquals(SamzaApplicationState.SamzaAppStatus.FAILED, state.status);
@@ -355,11 +356,11 @@ public void testInvalidNotificationsAreIgnored() throws Exception {
             new MapConfig(conf),
             state,
             new MetricsRegistryMap(),
-            manager
+        clusterResourceManager
     );
 
     MockContainerAllocator allocator = new MockContainerAllocator(
-            manager,
+        clusterResourceManager,
             conf,
             state);
     getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);
@@ -367,7 +368,7 @@ public void testInvalidNotificationsAreIgnored() throws Exception {
     Thread thread = new Thread(allocator);
     getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);
 
-    // Start the task manager
+    // Start the task clusterResourceManager
     taskManager.start();
 
     SamzaResource container = new SamzaResource(1, 1000, "abc", "id1");
@@ -388,6 +389,29 @@ public void testInvalidNotificationsAreIgnored() throws Exception {
     assertEquals(state.redundantNotifications.get(), 1);
   }
 
+  @Test
+  public void testRerequestOnAnyHostIfContainerStartFails() throws Exception {
+    state = new SamzaApplicationState(getJobModelManagerWithHostAffinity(1));
+    Map<String, String> configMap = new HashMap<>();
+    configMap.putAll(getConfig());
+
+    MockContainerAllocator allocator = new MockContainerAllocator(
+        clusterResourceManager,
+        new MapConfig(config),
+        state);
+
+    ContainerProcessManager manager = new ContainerProcessManager(config, state, new MetricsRegistryMap(), allocator,
+        clusterResourceManager);
+
+    manager.start();
+    SamzaResource resource = new SamzaResource(1, 1024, "abc", "resource-1");
+    state.pendingContainers.put("1", resource);
+    Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 1);
+    manager.onStreamProcessorLaunchFailure(resource, new Exception("cannot launch container!"));
+    Assert.assertEquals(clusterResourceManager.resourceRequests.size(), 2);
+    Assert.assertEquals(clusterResourceManager.resourceRequests.get(1).getHost(), ResourceRequestState.ANY_HOST);
+  }
+
   @Test
   public void testDuplicateNotificationsDoNotAffectJobHealth() throws Exception {
     Config conf = getConfig();
@@ -400,11 +424,11 @@ public void testDuplicateNotificationsDoNotAffectJobHealth() throws Exception {
             new MapConfig(conf),
             state,
             new MetricsRegistryMap(),
-            manager
+        clusterResourceManager
     );
 
     MockContainerAllocator allocator = new MockContainerAllocator(
-            manager,
+        clusterResourceManager,
             conf,
             state);
     getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);
@@ -424,6 +448,7 @@ public void testDuplicateNotificationsDoNotAffectJobHealth() throws Exception {
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
+    taskManager.onStreamProcessorLaunchSuccess(container1);
     assertEquals(0, allocator.getContainerRequestState().numPendingRequests());
 
     // Create container failure - with ContainerExitStatus.DISKS_FAILED
@@ -433,8 +458,8 @@ public void testDuplicateNotificationsDoNotAffectJobHealth() throws Exception {
     assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
     assertFalse(taskManager.shouldShutdown());
     assertFalse(state.jobHealthy.get());
-    assertEquals(2, manager.resourceRequests.size());
-    assertEquals(0, manager.releasedResources.size());
+    assertEquals(2, clusterResourceManager.resourceRequests.size());
+    assertEquals(0, clusterResourceManager.releasedResources.size());
     assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
 
     SamzaResource container2 = new SamzaResource(1, 1000, "abc", "id2");
@@ -444,14 +469,16 @@ public void testDuplicateNotificationsDoNotAffectJobHealth() throws Exception {
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
+    taskManager.onStreamProcessorLaunchSuccess(container2);
+
     assertTrue(state.jobHealthy.get());
 
     // Simulate a duplicate notification for container 1 with a different exit code
     taskManager.onResourceCompleted(new SamzaResourceStatus(container1.getResourceID(), "Disk failure", SamzaResourceStatus.PREEMPTED));
     // assert that a duplicate notification does not change metrics (including job health)
     assertEquals(state.redundantNotifications.get(), 1);
-    assertEquals(2, manager.resourceRequests.size());
-    assertEquals(0, manager.releasedResources.size());
+    assertEquals(2, clusterResourceManager.resourceRequests.size());
+    assertEquals(0, clusterResourceManager.releasedResources.size());
     assertTrue(state.jobHealthy.get());
   }
 
@@ -471,11 +498,11 @@ public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
 
     MockContainerAllocator allocator = new MockContainerAllocator(
-        manager,
+        clusterResourceManager,
         conf,
         state);
     getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);
@@ -483,7 +510,7 @@ public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
     Thread thread = new Thread(allocator);
     getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);
 
-    // Start the task manager
+    // Start the task clusterResourceManager
     taskManager.start();
     assertFalse(taskManager.shouldShutdown());
     assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
@@ -496,7 +523,7 @@ public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
       fail("timed out waiting for the containers to start");
     }
     assertEquals(0, allocator.getContainerRequestState().numPendingRequests());
-
+    taskManager.onStreamProcessorLaunchSuccess(container1);
     // Create container failure - with ContainerExitStatus.DISKS_FAILED
     taskManager.onResourceCompleted(new SamzaResourceStatus(container1.getResourceID(), "Disk failure", SamzaResourceStatus.DISK_FAIL));
 
@@ -504,8 +531,8 @@ public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
     assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
     assertFalse(taskManager.shouldShutdown());
     assertFalse(state.jobHealthy.get());
-    assertEquals(2, manager.resourceRequests.size());
-    assertEquals(0, manager.releasedResources.size());
+    assertEquals(2, clusterResourceManager.resourceRequests.size());
+    assertEquals(0, clusterResourceManager.releasedResources.size());
     assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
 
     SamzaResource container2 = new SamzaResource(1, 1000, "abc", "id2");
@@ -515,10 +542,11 @@ public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
+    taskManager.onStreamProcessorLaunchSuccess(container2);
 
     // Create container failure - with ContainerExitStatus.PREEMPTED
     taskManager.onResourceCompleted(new SamzaResourceStatus(container2.getResourceID(), "Preemption",  SamzaResourceStatus.PREEMPTED));
-    assertEquals(3, manager.resourceRequests.size());
+    assertEquals(3, clusterResourceManager.resourceRequests.size());
 
     // The above failure should trigger a container request
     assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
@@ -532,14 +560,15 @@ public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
     if (!allocator.awaitContainersStart(1, 2, TimeUnit.SECONDS)) {
       fail("timed out waiting for the containers to start");
     }
+    taskManager.onStreamProcessorLaunchSuccess(container3);
 
     // Create container failure - with ContainerExitStatus.ABORTED
     taskManager.onResourceCompleted(new SamzaResourceStatus(container3.getResourceID(), "Aborted", SamzaResourceStatus.ABORTED));
 
     // The above failure should trigger a container request
     assertEquals(1, allocator.getContainerRequestState().numPendingRequests());
-    assertEquals(4, manager.resourceRequests.size());
-    assertEquals(0, manager.releasedResources.size());
+    assertEquals(4, clusterResourceManager.resourceRequests.size());
+    assertEquals(0, clusterResourceManager.releasedResources.size());
     assertFalse(taskManager.shouldShutdown());
     assertFalse(state.jobHealthy.get());
     assertEquals(ResourceRequestState.ANY_HOST, allocator.getContainerRequestState().peekPendingRequest().getPreferredHost());
@@ -556,7 +585,7 @@ public void testAppMasterWithFwk() {
         new MapConfig(conf),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
     taskManager.start();
     SamzaResource container2 = new SamzaResource(1, 1024, "", "id0");
@@ -570,7 +599,7 @@ public void testAppMasterWithFwk() {
         new MapConfig(config),
         state,
         new MetricsRegistryMap(),
-        manager
+        clusterResourceManager
     );
     taskManager1.start();
     taskManager1.onResourceAllocated(container2);
diff --git a/samza-core/src/test/java/org/apache/samza/clustermanager/TestHostAwareContainerAllocator.java b/samza-core/src/test/java/org/apache/samza/clustermanager/TestHostAwareContainerAllocator.java
index 00198e9deb..6260b71bbf 100644
--- a/samza-core/src/test/java/org/apache/samza/clustermanager/TestHostAwareContainerAllocator.java
+++ b/samza-core/src/test/java/org/apache/samza/clustermanager/TestHostAwareContainerAllocator.java
@@ -18,7 +18,6 @@
  */
 package org.apache.samza.clustermanager;
 
-import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.HashMap;
 import java.util.Map;
@@ -203,61 +202,6 @@ public void testRequestContainers() throws Exception {
     assertEquals(1, requestsMap.get(ResourceRequestState.ANY_HOST).get());
   }
 
-  /**
-   * If the container fails to start e.g because it fails to connect to a NM on a host that
-   * is down, the allocator should request a new container on a different host.
-   */
-  @Test
-  public void testRerequestOnAnyHostIfContainerStartFails() throws Exception {
-
-    final SamzaResource container = new SamzaResource(1, 1024, "2", "id0");
-    final SamzaResource container1 = new SamzaResource(1, 1024, "1", "id1");
-    manager.nextException = new IOException("Cant connect to RM");
-
-    // Set up our final asserts before starting the allocator thread
-    MockContainerListener listener = new MockContainerListener(2, 1, 2, 0, null, new Runnable() {
-      @Override
-      public void run() {
-        // The failed container should be released. The successful one should not.
-        assertNotNull(manager.releasedResources);
-        assertEquals(1, manager.releasedResources.size());
-        assertTrue(manager.releasedResources.contains(container));
-      }
-    },
-        new Runnable() {
-          @Override
-          public void run() {
-            // Test that the first request assignment had a preferred host and the retry didn't
-            assertEquals(2, requestState.assignedRequests.size());
-
-            SamzaResourceRequest request = requestState.assignedRequests.remove();
-            assertEquals("0", request.getContainerID());
-            assertEquals("2", request.getPreferredHost());
-
-            request = requestState.assignedRequests.remove();
-            assertEquals("0", request.getContainerID());
-            assertEquals("ANY_HOST", request.getPreferredHost());
-
-            // This routine should be called after the retry is assigned, but before it's started.
-            // So there should still be 1 container needed.
-            assertEquals(1, state.neededContainers.get());
-          }
-        }, null
-    );
-    state.neededContainers.set(1);
-    requestState.registerContainerListener(listener);
-
-    // Only request 1 container and we should see 2 assignments in the assertions above (because of the retry)
-    containerAllocator.requestResource("0", "2");
-    containerAllocator.addResource(container1);
-    containerAllocator.addResource(container);
-
-    allocatorThread.start();
-
-    listener.verify();
-  }
-
-
   /**
    * Handles expired requests correctly and assigns ANY_HOST
    */
diff --git a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImpl.java b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImpl.java
index 904367b438..a4f0354bb7 100644
--- a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImpl.java
+++ b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImpl.java
@@ -21,6 +21,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Set;
+
 import org.apache.samza.config.Config;
 import org.apache.samza.container.TaskContextImpl;
 import org.apache.samza.metrics.Counter;
diff --git a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnAppState.java b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnAppState.java
index db67de6ddf..fb9a9c24e6 100644
--- a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnAppState.java
+++ b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnAppState.java
@@ -49,7 +49,9 @@ public class YarnAppState {
   * Modified by both the AMRMCallbackThread and the ContainerAllocator thread
   */
 
-  public Map<String, YarnContainer> runningYarnContainers = new ConcurrentHashMap<String, YarnContainer>()  ;
+  public Map<String, YarnContainer> runningYarnContainers = new ConcurrentHashMap<String, YarnContainer>();
+
+  public Map<String, YarnContainer> pendingYarnContainers = new ConcurrentHashMap<String, YarnContainer>();
 
   public ConcurrentMap<String, ContainerStatus> failedContainersStatus = new ConcurrentHashMap<String, ContainerStatus>();
 
diff --git a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java
index 96a4488fef..9be847539a 100644
--- a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java
+++ b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java
@@ -19,30 +19,46 @@
 
 package org.apache.samza.job.yarn;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.*;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
 import org.apache.hadoop.yarn.api.records.*;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.NMClient;
 import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.samza.SamzaException;
 import org.apache.samza.clustermanager.*;
 import org.apache.samza.clustermanager.SamzaApplicationState;
 import org.apache.samza.clustermanager.SamzaContainerLaunchException;
 import org.apache.samza.config.ClusterManagerConfig;
 import org.apache.samza.config.Config;
+import org.apache.samza.config.JobConfig;
 import org.apache.samza.config.ShellCommandConfig;
 import org.apache.samza.config.YarnConfig;
 import org.apache.samza.coordinator.JobModelManager;
 import org.apache.samza.job.CommandBuilder;
 import org.apache.samza.metrics.MetricsRegistryMap;
+import org.apache.samza.util.Util;
 import org.apache.samza.util.hadoop.HttpFileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -61,7 +77,7 @@
  *
  */
 
-public class YarnClusterResourceManager extends ClusterResourceManager implements AMRMClientAsync.CallbackHandler {
+public class YarnClusterResourceManager extends ClusterResourceManager implements AMRMClientAsync.CallbackHandler, NMClientAsync.CallbackHandler {
 
   private final String INVALID_YARN_CONTAINER_ID = "-1";
 
@@ -70,15 +86,10 @@ public class YarnClusterResourceManager extends ClusterResourceManager implement
    */
   private final AMRMClientAsync<AMRMClient.ContainerRequest> amClient;
 
-  /**
-   * A helper class to launch Yarn containers.
-   */
-  private final YarnContainerRunner yarnContainerRunner;
-
   /**
    * Configuration and state specific to Yarn.
    */
-  private final YarnConfiguration hConfig;
+  private final YarnConfiguration yarnConfiguration;
   private final YarnAppState state;
 
   /**
@@ -100,12 +111,16 @@ public class YarnClusterResourceManager extends ClusterResourceManager implement
   private final ConcurrentHashMap<SamzaResource, Container> allocatedResources = new ConcurrentHashMap<>();
   private final ConcurrentHashMap<SamzaResourceRequest, AMRMClient.ContainerRequest> requestsMap = new ConcurrentHashMap<>();
 
+  private final ConcurrentHashMap<ContainerId, Container> containersPendingStartup = new ConcurrentHashMap<>();
+
   private final SamzaAppMasterMetrics metrics;
 
   final AtomicBoolean started = new AtomicBoolean(false);
   private final Object lock = new Object();
+  private final NMClientAsync nmClientAsync;
 
   private static final Logger log = LoggerFactory.getLogger(YarnClusterResourceManager.class);
+  private final Config config;
 
   /**
    * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
@@ -116,15 +131,15 @@ public class YarnClusterResourceManager extends ClusterResourceManager implement
    */
   public YarnClusterResourceManager(Config config, JobModelManager jobModelManager, ClusterResourceManager.Callback callback, SamzaApplicationState samzaAppState ) {
     super(callback);
-    hConfig = new YarnConfiguration();
-    hConfig.set("fs.http.impl", HttpFileSystem.class.getName());
+    yarnConfiguration = new YarnConfiguration();
+    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
 
     // Use the Samza job config "fs.<scheme>.impl" and "fs.<scheme>.impl.*" for YarnConfiguration
     FileSystemImplConfig fsImplConfig = new FileSystemImplConfig(config);
     fsImplConfig.getSchemes().forEach(
         scheme -> {
           fsImplConfig.getSchemeConfig(scheme).forEach(
-              (confKey, confValue) -> hConfig.set(confKey, confValue)
+              (confKey, confValue) -> yarnConfiguration.set(confKey, confValue)
           );
         }
     );
@@ -143,6 +158,7 @@ public YarnClusterResourceManager(Config config, JobModelManager jobModelManager
     int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
     YarnConfig yarnConfig = new YarnConfig(config);
     this.yarnConfig = yarnConfig;
+    this.config = config;
     int interval = yarnConfig.getAMPollIntervalMs();
 
     //Instantiate the AM Client.
@@ -151,7 +167,7 @@ public YarnClusterResourceManager(Config config, JobModelManager jobModelManager
     this.state = new YarnAppState(-1, containerId, nodeHostString, nodePort, nodeHttpPort);
 
     log.info("Initialized YarnAppState: {}", state.toString());
-    this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry, hConfig);
+    this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry, yarnConfiguration);
 
     log.info("ContainerID str {}, Nodehost  {} , Nodeport  {} , NodeHttpport {}", new Object [] {containerIdStr, nodeHostString, nodePort, nodeHttpPort});
     ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
@@ -162,8 +178,8 @@ public YarnClusterResourceManager(Config config, JobModelManager jobModelManager
         state,
         amClient
     );
+    this.nmClientAsync = NMClientAsync.createNMClientAsync(this);
 
-    yarnContainerRunner = new YarnContainerRunner(config, hConfig);
   }
 
   /**
@@ -179,8 +195,10 @@ public void start() {
     metrics.start();
     service.onInit();
     log.info("Starting YarnContainerManager.");
-    amClient.init(hConfig);
+    amClient.init(yarnConfiguration);
     amClient.start();
+    nmClientAsync.init(yarnConfiguration);
+    nmClientAsync.start();
     lifecycle.onInit();
 
     if(lifecycle.shouldShutdown()) {
@@ -262,19 +280,22 @@ public void releaseResources(SamzaResource resource) {
    */
 
   @Override
-  public void launchStreamProcessor(SamzaResource resource, CommandBuilder builder) throws SamzaContainerLaunchException {
+  public void launchStreamProcessor(SamzaResource resource, CommandBuilder builder) {
     String containerIDStr = builder.buildEnvironment().get(ShellCommandConfig.ENV_CONTAINER_ID());
-    log.info("Received launch request for {} on hostname {}", containerIDStr , resource.getHost());
-
+    log.info("Received launch request for {} on hostname {}", containerIDStr, resource.getHost());
     synchronized (lock) {
-      Container container = allocatedResources.get(resource);
-      if (container == null) {
-        log.info("Resource {} already released. ", resource);
-        return;
-      }
+      try {
+        Container container = allocatedResources.get(resource);
+        if (container == null) {
+          log.info("Resource {} already released. ", resource);
+          return;
+        }
 
-      state.runningYarnContainers.put(containerIDStr, new YarnContainer(container));
-      yarnContainerRunner.runContainer(containerIDStr, container, builder);
+        runContainer(containerIDStr, container, builder);
+      } catch (Throwable t) {
+        log.error("Error in launching stream processor:", t);
+        clusterManagerCallback.onStreamProcessorLaunchFailure(resource, t);
+      }
     }
   }
 
@@ -338,6 +359,8 @@ public void stop(SamzaApplicationState.SamzaAppStatus status) {
     lifecycle.onShutdown(status);
     amClient.stop();
     log.info("Stopping the AM service " );
+    nmClientAsync.stop();
+    log.info("Stopping the NM service " );
     service.onShutdown();
     metrics.stop();
 
@@ -358,7 +381,7 @@ private void cleanupStagingDir() {
 
       FileSystem fs = null;
       try {
-        fs = FileSystem.get(hConfig);
+        fs = FileSystem.get(yarnConfiguration);
       } catch (IOException e) {
         log.error("Unable to clean up file system: {}", e);
         return;
@@ -454,4 +477,243 @@ public void onError(Throwable e) {
     clusterManagerCallback.onError(e);
   }
 
+  @Override
+  public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) {
+    log.info("Received a containerStart notification from the NodeManager for container: {} ", containerId);
+    String samzaContainerId = getPendingSamzaContainerId(containerId);
+
+    if (samzaContainerId != null) {
+      // 1. Move the container from pending to running state
+      final YarnContainer container = state.pendingYarnContainers.remove(samzaContainerId);
+      log.info("Samza containerId:{} has started", samzaContainerId);
+
+      state.runningYarnContainers.put(samzaContainerId, container);
+
+      // 2. Invoke the success callback.
+      SamzaResource resource = new SamzaResource(container.resource().getVirtualCores(),
+          container.resource().getMemory(), container.nodeId().getHost(), containerId.toString());
+      clusterManagerCallback.onStreamProcessorLaunchSuccess(resource);
+    } else {
+      log.info("Got an invalid notification from YARN for container: {}", containerId);
+    }
+  }
+
+  @Override
+  public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) {
+    log.info("Got a status from the NodeManager. Container: {} Status: {}", containerId, containerStatus.getState());
+  }
+
+  @Override
+  public void onContainerStopped(ContainerId containerId) {
+    log.info("Got a notification from the NodeManager for a stopped container. ContainerId: {}", containerId);
+  }
+
+  @Override
+  public void onStartContainerError(ContainerId containerId, Throwable t) {
+    log.error(String.format("Container: %s could not start.", containerId), t);
+
+    Container container = containersPendingStartup.remove(containerId);
+
+    if (container != null) {
+      SamzaResource resource = new SamzaResource(container.getResource().getVirtualCores(),
+          container.getResource().getMemory(), container.getNodeId().getHost(), containerId.toString());
+      log.info("Invoking failure callback for container: {}", containerId);
+      clusterManagerCallback.onStreamProcessorLaunchFailure(resource, new SamzaContainerLaunchException(t));
+    } else {
+      log.info("Got an invalid notification for container: {}", containerId);
+    }
+  }
+
+  @Override
+  public void onGetContainerStatusError(ContainerId containerId, Throwable t) {
+    log.info("Got an error on getContainerStatus from the NodeManager. ContainerId: {}. Error: {}", containerId, t);
+  }
+
+  @Override
+  public void onStopContainerError(ContainerId containerId, Throwable t) {
+    log.info("Got an error when stopping container from the NodeManager. ContainerId: {}. Error: {}", containerId, t);
+  }
+
+  /**
+   * Runs a process as specified by the command builder on the container.
+   * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process)
+   * @param container the samza container to run.
+   * @param cmdBuilder the command builder that encapsulates the command, and the context
+   *
+   */
+  public void runContainer(String samzaContainerId, Container container, CommandBuilder cmdBuilder) throws IOException {
+    String containerIdStr = ConverterUtils.toString(container.getId());
+    log.info("Got available container ID ({}) for container: {}", samzaContainerId, container);
+
+    // check if we have framework path specified. If yes - use it, if not use default ./__package/
+    String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries
+    String cmdPath = "./__package/";
+
+    String fwkPath = JobConfig.getFwkPath(this.config);
+    if(fwkPath != null && (! fwkPath.isEmpty())) {
+      cmdPath = fwkPath;
+      jobLib = "export JOB_LIB_DIR=./__package/lib";
+    }
+    log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib);
+    cmdBuilder.setCommandPath(cmdPath);
+
+
+    String command = cmdBuilder.buildCommand();
+    log.info("Container ID {} using command {}", samzaContainerId, command);
+
+    Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
+    env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString()));
+    printContainerEnvironmentVariables(samzaContainerId, env);
+
+    log.info("Samza FWK path: " + command + "; env=" + env);
+
+    Path packagePath = new Path(yarnConfig.getPackagePath());
+    log.info("Starting container ID {} using package path {}", samzaContainerId, packagePath);
+    state.pendingYarnContainers.put(samzaContainerId, new YarnContainer(container));
+
+    startContainer(
+        packagePath,
+        container,
+        env,
+        getFormattedCommand(
+            ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+            jobLib,
+            command,
+            ApplicationConstants.STDOUT,
+            ApplicationConstants.STDERR)
+    );
+
+
+    log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).",
+        new Object[]{
+            samzaContainerId,
+            containerIdStr,
+            container.getNodeId().getHost(),
+            container.getNodeHttpAddress(),
+            containerIdStr}
+    );
+
+    log.info("Started container ID {}", samzaContainerId);
+  }
+
+  /**
+   *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
+   *    specified by packagePath.
+   */
+  private void startContainer(Path packagePath,
+                              Container container,
+                              Map<String, String> env,
+                              final String cmd) throws IOException {
+    log.info("Starting container {} {} {} {}",
+        new Object[]{packagePath, container, env, cmd});
+
+    LocalResource packageResource = Records.newRecord(LocalResource.class);
+    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
+    FileStatus fileStatus;
+    fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
+    packageResource.setResource(packageUrl);
+    log.info("Set package resource in YarnContainerRunner for {}", packageUrl);
+    packageResource.setSize(fileStatus.getLen());
+    packageResource.setTimestamp(fileStatus.getModificationTime());
+    packageResource.setType(LocalResourceType.ARCHIVE);
+    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);
+
+    ByteBuffer allTokens;
+    // copy tokens to start the container
+    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
+    DataOutputBuffer dob = new DataOutputBuffer();
+    credentials.writeTokenStorageToStream(dob);
+
+    // now remove the AM->RM token so that containers cannot access it
+    Iterator iter = credentials.getAllTokens().iterator();
+    while (iter.hasNext()) {
+      TokenIdentifier token = ((org.apache.hadoop.security.token.Token) iter.next()).decodeIdentifier();
+      if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
+        iter.remove();
+      }
+    }
+    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+
+    Map<String, LocalResource> localResourceMap = new HashMap<>();
+    localResourceMap.put("__package", packageResource);
+
+    // include the resources from the universal resource configurations
+    LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration);
+    localResourceMap.putAll(resourceMapper.getResourceMap());
+
+    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
+    context.setEnvironment(env);
+    context.setTokens(allTokens.duplicate());
+    context.setCommands(new ArrayList<String>() {{add(cmd);}});
+    context.setLocalResources(localResourceMap);
+
+    log.debug("Setting localResourceMap to {}", localResourceMap);
+    log.debug("Setting context to {}", context);
+
+    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
+    startContainerRequest.setContainerLaunchContext(context);
+
+    log.info("Making an async start request for container {}", container);
+    nmClientAsync.startContainerAsync(container, context);
+  }
+
+  /**
+   * @param samzaContainerId  the Samza container Id for logging purposes.
+   * @param env               the Map of environment variables to their respective values.
+   */
+  private void printContainerEnvironmentVariables(String samzaContainerId, Map<String, String> env) {
+    StringBuilder sb = new StringBuilder();
+    for (Map.Entry<String, String> entry : env.entrySet()) {
+      sb.append(String.format("\n%s=%s", entry.getKey(), entry.getValue()));
+    }
+    log.info("Container ID {} using environment variables: {}", samzaContainerId, sb.toString());
+  }
+
+
+  /**
+   * Gets the environment variables from the specified {@link CommandBuilder} and escapes certain characters.
+   *
+   * @param cmdBuilder        the command builder containing the environment variables.
+   * @return                  the map containing the escaped environment variables.
+   */
+  private Map<String, String> getEscapedEnvironmentVariablesMap(CommandBuilder cmdBuilder) {
+    Map<String, String> env = new HashMap<String, String>();
+    for (Map.Entry<String, String> entry : cmdBuilder.buildEnvironment().entrySet()) {
+      String escapedValue = Util.envVarEscape(entry.getValue());
+      env.put(entry.getKey(), escapedValue);
+    }
+    return env;
+  }
+
+
+  private String getFormattedCommand(String logDirExpansionVar,
+                                     String jobLib,
+                                     String command,
+                                     String stdOut,
+                                     String stdErr) {
+    if (!jobLib.isEmpty()) {
+      jobLib = "&& " + jobLib; // add job's libraries exported to an env variable
+    }
+
+    return String
+        .format("export SAMZA_LOG_DIR=%s %s && ln -sfn %s logs && exec %s 1>logs/%s 2>logs/%s", logDirExpansionVar,
+            jobLib, logDirExpansionVar, command, stdOut, stdErr);
+  }
+
+  /**
+   * Returns the Id of the Samza container that corresponds to the provided Yarn {@link ContainerId}
+   * @param containerId the Yarn ContainerId
+   * @return the id of the Samza container corresponding to the {@link ContainerId} that is pending launch
+   */
+  private String getPendingSamzaContainerId(ContainerId containerId) {
+    for (String samzaContainerId: state.pendingYarnContainers.keySet()) {
+      YarnContainer yarnContainer = state.pendingYarnContainers.get(samzaContainerId);
+      if (yarnContainer.id().equals(containerId)) {
+        return samzaContainerId;
+      }
+    }
+    return null;
+  }
+
+
 }
diff --git a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnContainerRunner.java b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnContainerRunner.java
deleted file mode 100644
index cdcf2d11c8..0000000000
--- a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnContainerRunner.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.samza.job.yarn;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.records.*;
-import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.samza.clustermanager.SamzaContainerLaunchException;
-import org.apache.samza.config.Config;
-import org.apache.samza.config.JobConfig;
-import org.apache.samza.config.ShellCommandConfig;
-import org.apache.samza.config.YarnConfig;
-import org.apache.samza.job.CommandBuilder;
-import org.apache.samza.util.Util;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * A Helper class to run container processes on Yarn. This encapsulates quite a bit of YarnContainer
- * boiler plate.
- */
-public class YarnContainerRunner {
-  private static final Logger log = LoggerFactory.getLogger(YarnContainerRunner.class);
-
-  private final Config config;
-  private final YarnConfiguration yarnConfiguration;
-
-  private final NMClient nmClient;
-  private final YarnConfig yarnConfig;
-
-  /**
-   * Create a new Runner from a Config.
-   * @param config to instantiate the runner with
-   * @param yarnConfiguration the yarn config for the cluster to connect to.
-   */
-
-  public YarnContainerRunner(Config config,
-                             YarnConfiguration yarnConfiguration) {
-    this.config = config;
-    this.yarnConfiguration = yarnConfiguration;
-
-    this.nmClient = NMClient.createNMClient();
-    nmClient.init(this.yarnConfiguration);
-
-    this.yarnConfig = new YarnConfig(config);
-  }
-
-  /**
-   * Runs a process as specified by the command builder on the container.
-   * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process)
-   * @param container the samza container to run.
-   * @param cmdBuilder the command builder that encapsulates the command, and the context
-   *
-   * @throws SamzaContainerLaunchException  when there's an exception in submitting the request to the RM.
-   *
-   */
-  public void runContainer(String samzaContainerId, Container container, CommandBuilder cmdBuilder) throws SamzaContainerLaunchException {
-    String containerIdStr = ConverterUtils.toString(container.getId());
-    log.info("Got available container ID ({}) for container: {}", samzaContainerId, container);
-
-    // check if we have framework path specified. If yes - use it, if not use default ./__package/
-    String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries
-    String cmdPath = "./__package/";
-
-    String fwkPath = JobConfig.getFwkPath(config);
-    if(fwkPath != null && (! fwkPath.isEmpty())) {
-      cmdPath = fwkPath;
-      jobLib = "export JOB_LIB_DIR=./__package/lib";
-    }
-    log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib);
-    cmdBuilder.setCommandPath(cmdPath);
-
-
-    String command = cmdBuilder.buildCommand();
-    log.info("Container ID {} using command {}", samzaContainerId, command);
-
-    Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
-    env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString()));
-    printContainerEnvironmentVariables(samzaContainerId, env);
-
-    log.info("Samza FWK path: " + command + "; env=" + env);
-
-    Path packagePath = new Path(yarnConfig.getPackagePath());
-    log.info("Starting container ID {} using package path {}", samzaContainerId, packagePath);
-
-    startContainer(
-        packagePath,
-        container,
-        env,
-        getFormattedCommand(
-            ApplicationConstants.LOG_DIR_EXPANSION_VAR,
-            jobLib,
-            command,
-            ApplicationConstants.STDOUT,
-            ApplicationConstants.STDERR)
-    );
-
-
-    log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).",
-        new Object[]{
-            samzaContainerId,
-            containerIdStr,
-            container.getNodeId().getHost(),
-            container.getNodeHttpAddress(),
-            containerIdStr}
-    );
-
-      log.info("Started container ID {}", samzaContainerId);
-  }
-
-  /**
-   *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
-   *    specified by packagePath.
-   */
-  private void startContainer(Path packagePath,
-                                Container container,
-                                Map<String, String> env,
-                                final String cmd) throws SamzaContainerLaunchException {
-    log.info("starting container {} {} {} {}",
-        new Object[]{packagePath, container, env, cmd});
-
-    // TODO: SAMZA-1144 remove the customized approach for package resource and use the common one.
-    // But keep it now for backward compatibility.
-    // set the local package so that the containers and app master are provisioned with it
-    LocalResource packageResource = Records.newRecord(LocalResource.class);
-    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
-    FileStatus fileStatus;
-    try {
-      fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
-    } catch (IOException ioe) {
-      log.error("IO Exception when accessing the package status from the filesystem", ioe);
-      throw new SamzaContainerLaunchException("IO Exception when accessing the package status from the filesystem");
-    }
-
-    packageResource.setResource(packageUrl);
-    log.info("set package Resource in YarnContainerRunner for {}", packageUrl);
-    packageResource.setSize(fileStatus.getLen());
-    packageResource.setTimestamp(fileStatus.getModificationTime());
-    packageResource.setType(LocalResourceType.ARCHIVE);
-    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);
-
-    ByteBuffer allTokens;
-    // copy tokens (copied from dist shell example)
-    try {
-      Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
-      DataOutputBuffer dob = new DataOutputBuffer();
-      credentials.writeTokenStorageToStream(dob);
-
-      // now remove the AM->RM token so that containers cannot access it
-      Iterator iter = credentials.getAllTokens().iterator();
-      while (iter.hasNext()) {
-        TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
-        if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
-          iter.remove();
-        }
-      }
-      allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
-
-    } catch (IOException ioe) {
-      log.error("IOException when writing credentials.", ioe);
-      throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer");
-    }
-
-    Map<String, LocalResource> localResourceMap = new HashMap<>();
-    localResourceMap.put("__package", packageResource);
-
-    // include the resources from the universal resource configurations
-    LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration);
-    localResourceMap.putAll(resourceMapper.getResourceMap());
-
-    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
-    context.setEnvironment(env);
-    context.setTokens(allTokens.duplicate());
-    context.setCommands(new ArrayList<String>() {{add(cmd);}});
-    context.setLocalResources(localResourceMap);
-
-    log.debug("setting localResourceMap to {}", localResourceMap);
-    log.debug("setting context to {}", context);
-
-    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
-    startContainerRequest.setContainerLaunchContext(context);
-    try {
-      nmClient.startContainer(container, context);
-    } catch (YarnException ye) {
-      log.error("Received YarnException when starting container: " + container.getId(), ye);
-      throw new SamzaContainerLaunchException("Received YarnException when starting container: " + container.getId(), ye);
-    } catch (IOException ioe) {
-      log.error("Received IOException when starting container: " + container.getId(), ioe);
-      throw new SamzaContainerLaunchException("Received IOException when starting container: " + container.getId(), ioe);
-    }
-  }
-
-
-  /**
-   * @param samzaContainerId  the Samza container Id for logging purposes.
-   * @param env               the Map of environment variables to their respective values.
-   */
-  private void printContainerEnvironmentVariables(String samzaContainerId, Map<String, String> env) {
-    StringBuilder sb = new StringBuilder();
-    for (Map.Entry<String, String> entry : env.entrySet()) {
-      sb.append(String.format("\n%s=%s", entry.getKey(), entry.getValue()));
-    }
-    log.info("Container ID {} using environment variables: {}", samzaContainerId, sb.toString());
-  }
-
-
-  /**
-   * Gets the environment variables from the specified {@link CommandBuilder} and escapes certain characters.
-   *
-   * @param cmdBuilder        the command builder containing the environment variables.
-   * @return                  the map containing the escaped environment variables.
-   */
-  private Map<String, String> getEscapedEnvironmentVariablesMap(CommandBuilder cmdBuilder) {
-    Map<String, String> env = new HashMap<String, String>();
-    for (Map.Entry<String, String> entry : cmdBuilder.buildEnvironment().entrySet()) {
-      String escapedValue = Util.envVarEscape(entry.getValue());
-      env.put(entry.getKey(), escapedValue);
-    }
-    return env;
-  }
-
-
-  private String getFormattedCommand(String logDirExpansionVar,
-                                     String jobLib,
-                                     String command,
-                                     String stdOut,
-                                     String stdErr) {
-    if (!jobLib.isEmpty()) {
-      jobLib = "&& " + jobLib; // add job's libraries exported to an env variable
-    }
-
-    return String
-        .format("export SAMZA_LOG_DIR=%s %s && ln -sfn %s logs && exec %s 1>logs/%s 2>logs/%s", logDirExpansionVar,
-            jobLib, logDirExpansionVar, command, stdOut, stdErr);
-  }
-}

From f8d91425e04ae46d5d5bfc72f18663f3d1afdc1f Mon Sep 17 00:00:00 2001
From: Xinyu Liu <xinyuliu.us@gmail.com>
Date: Wed, 3 Jan 2018 09:56:23 -0800
Subject: [PATCH 24/36] SAMZA-1550: Doc for 0.14.0 release

Docs update for both master and 0.14.0 branch.

Author: xiliu <xiliu@xiliu-ld1.linkedin.biz>

Reviewers: Yi Pan <nickpan47@gmail.com>

Closes #396 from xinyuiscool/SAMZA-1550
---
 docs/_layouts/default.html                    |  1 +
 docs/archive/index.html                       |  8 ++++++++
 .../versioned/deploy-samza-job-from-hdfs.md   |  2 +-
 .../versioned/deploy-samza-to-CDH.md          |  4 ++--
 .../versioned/remote-debugging-samza.md       |  2 +-
 .../versioned/run-in-multi-node-yarn.md       |  4 ++--
 docs/startup/download/index.md                | 19 ++++++++++---------
 .../releases/versioned/release-notes.md       | 13 ++++++++++---
 8 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html
index c48073cee6..75976570bc 100644
--- a/docs/_layouts/default.html
+++ b/docs/_layouts/default.html
@@ -103,6 +103,7 @@ <h1><i class="fa fa-code"></i> Contribute</h1>
             <h1><i class="fa fa-history"></i> Archive</h1>
             <ul>
               <li><a href="/archive/index.html#latest">latest</a></li>
+              <li><a href="/archive/index.html#14">0.14</a></li>
               <li><a href="/archive/index.html#13">0.13</a></li>
               <li><a href="/archive/index.html#12">0.12</a></li>
               <li><a href="/archive/index.html#11">0.11</a></li>
diff --git a/docs/archive/index.html b/docs/archive/index.html
index 6b6e74abac..0df2c205e8 100644
--- a/docs/archive/index.html
+++ b/docs/archive/index.html
@@ -27,6 +27,14 @@ <h4 id="latest">Latest Release</h4>
   <li><a href="../startup/hello-samza/latest">Hello Samza</a></li>
 </ul>
 
+<h4 id="14">0.14 Release</h4>
+
+<ul class="documentation-list">
+    <li><a href="../learn/documentation/0.14">Documentation</a></li>
+    <li><a href="../learn/tutorials/0.14">Tutorials</a></li>
+    <li><a href="../startup/hello-samza/0.14">Hello Samza</a></li>
+</ul>
+
 <h4 id="13">0.13 Release</h4>
 
 <ul class="documentation-list">
diff --git a/docs/learn/tutorials/versioned/deploy-samza-job-from-hdfs.md b/docs/learn/tutorials/versioned/deploy-samza-job-from-hdfs.md
index 13eba67852..acbc70b1bf 100644
--- a/docs/learn/tutorials/versioned/deploy-samza-job-from-hdfs.md
+++ b/docs/learn/tutorials/versioned/deploy-samza-job-from-hdfs.md
@@ -24,7 +24,7 @@ This tutorial uses [hello-samza](../../../startup/hello-samza/{{site.version}}/)
 ### Upload the package
 
 {% highlight bash %}
-hadoop fs -put ./target/hello-samza-0.13.1-dist.tar.gz /path/for/tgz
+hadoop fs -put ./target/hello-samza-0.14.0-dist.tar.gz /path/for/tgz
 {% endhighlight %}
 
 ### Add HDFS configuration
diff --git a/docs/learn/tutorials/versioned/deploy-samza-to-CDH.md b/docs/learn/tutorials/versioned/deploy-samza-to-CDH.md
index e5ab8cf400..296203bcdb 100644
--- a/docs/learn/tutorials/versioned/deploy-samza-to-CDH.md
+++ b/docs/learn/tutorials/versioned/deploy-samza-to-CDH.md
@@ -34,7 +34,7 @@ mvn clean package -Dhadoop.version=cdh5.4.0
 There are a few ways of uploading the package to the cluster's HDFS. If you do not have the job package in your cluster, **scp** from you local machine to the cluster. Then run
 
 {% highlight bash %}
-hadoop fs -put path/to/hello-samza-0.13.1-dist.tar.gz /path/for/tgz
+hadoop fs -put path/to/hello-samza-0.14.0-dist.tar.gz /path/for/tgz
 {% endhighlight %}
 
 ### Get Deploying Scripts
@@ -42,7 +42,7 @@ hadoop fs -put path/to/hello-samza-0.13.1-dist.tar.gz /path/for/tgz
 Untar the job package (assume you will run from the current directory)
 
 {% highlight bash %}
-tar -xvf path/to/samza-job-package-0.13.1-dist.tar.gz -C ./
+tar -xvf path/to/samza-job-package-0.14.0-dist.tar.gz -C ./
 {% endhighlight %}
 
 ### Add Package Path to Properties File
diff --git a/docs/learn/tutorials/versioned/remote-debugging-samza.md b/docs/learn/tutorials/versioned/remote-debugging-samza.md
index 4c7f10eaaa..dfa57cb530 100644
--- a/docs/learn/tutorials/versioned/remote-debugging-samza.md
+++ b/docs/learn/tutorials/versioned/remote-debugging-samza.md
@@ -83,7 +83,7 @@ Once the grid starts, you can start the wikipedia-feed Samza job.
 {% highlight bash %}
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.13.1-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.0-dist.tar.gz -C deploy/samza
 deploy/samza/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/wikipedia-feed.properties
 {% endhighlight %}
 
diff --git a/docs/learn/tutorials/versioned/run-in-multi-node-yarn.md b/docs/learn/tutorials/versioned/run-in-multi-node-yarn.md
index bb908ea553..f6146ad382 100644
--- a/docs/learn/tutorials/versioned/run-in-multi-node-yarn.md
+++ b/docs/learn/tutorials/versioned/run-in-multi-node-yarn.md
@@ -145,7 +145,7 @@ vi src/main/config/wikipedia-feed.properties
 Change the yarn.package.path property to be:
 
 {% highlight jproperties %}
-yarn.package.path=http://yourHostname:8000/target/hello-samza-0.13.1-dist.tar.gz
+yarn.package.path=http://yourHostname:8000/target/hello-samza-0.14.0-dist.tar.gz
 {% endhighlight %}
 
 3\. Compile hello-samza.
@@ -153,7 +153,7 @@ yarn.package.path=http://yourHostname:8000/target/hello-samza-0.13.1-dist.tar.gz
 {% highlight bash %}
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.13.1-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.0-dist.tar.gz -C deploy/samza
 {% endhighlight %}
 
 4\. Deploy Samza job package to Http server..
diff --git a/docs/startup/download/index.md b/docs/startup/download/index.md
index 5dcf38022b..39492c040c 100644
--- a/docs/startup/download/index.md
+++ b/docs/startup/download/index.md
@@ -31,10 +31,11 @@ Starting from 2016, Samza will begin requiring JDK8 or higher. Please see [this
  
  Samza tools package contains command line tools that user can run to use Samza and it's input/output systems. 
  
- * [samza-tools-0.14.0.tgz](tbd)
+ * [samza-tools-0.14.0.tgz](http://www-us.apache.org/dist/samza/0.14.0/samza-tools-0.14.0.tgz)
 
 ### Source Releases
 
+ * [samza-sources-0.14.0.tgz](http://www.apache.org/dyn/closer.lua/samza/0.14.0)
  * [samza-sources-0.13.1.tgz](http://www.apache.org/dyn/closer.lua/samza/0.13.1)
  * [samza-sources-0.13.0.tgz](http://www.apache.org/dyn/closer.lua/samza/0.13.0)
  * [samza-sources-0.12.0.tgz](http://www.apache.org/dyn/closer.lua/samza/0.12.0)
@@ -59,12 +60,12 @@ A Maven-based Samza project can pull in all required dependencies Samza dependen
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-api</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
 </dependency>
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-core_2.11</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 <dependency>
@@ -72,37 +73,37 @@ A Maven-based Samza project can pull in all required dependencies Samza dependen
   <artifactId>samza-shell</artifactId>
   <classifier>dist</classifier>
   <type>tgz</type>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-yarn_2.11</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-kv_2.11</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-kv-rocksdb_2.11</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-kv-inmemory_2.11</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 <dependency>
   <groupId>org.apache.samza</groupId>
   <artifactId>samza-kafka_2.11</artifactId>
-  <version>0.13.1</version>
+  <version>0.14.0</version>
   <scope>runtime</scope>
 </dependency>
 {% endhighlight %}
diff --git a/docs/startup/releases/versioned/release-notes.md b/docs/startup/releases/versioned/release-notes.md
index 460cbda7b3..f83978d291 100644
--- a/docs/startup/releases/versioned/release-notes.md
+++ b/docs/startup/releases/versioned/release-notes.md
@@ -31,6 +31,7 @@ title: Release Notes
 
 {% if site.version != "latest" %}
 ## Download
+All Samza JARs are published through [Apache's Maven repository](https://repository.apache.org/content/groups/public/org/apache/samza/). See [here](../download/index.html) for more details.
 
 ## Source Release
 [samza-sources-{{site.version}}.tgz](http://www.apache.org/dyn/closer.lua/samza/{{site.version}}.*)
@@ -42,8 +43,14 @@ title: Release Notes
 
 ## Release Notes
 <!-- Add notes on new features, modified behavior of existing features, operational/performance improvements, new tools etc -->
-
-<TBD>
+* [SAMZA-1510](https://issues.apache.org/jira/browse/SAMZA-1510) - Samza SQL
+* [SAMZA-1438](https://issues.apache.org/jira/browse/SAMZA-1438) - Producer and consumer for Azure EventHubs
+* [SAMZA-1515](https://issues.apache.org/jira/browse/SAMZA-1515) - Kinesis consumer
+* [SAMZA-1486](https://issues.apache.org/jira/browse/SAMZA-1486) - Checkpoint provider for Azure tables
+* [SAMZA-1421](https://issues.apache.org/jira/browse/SAMZA-1421) - Support for durable state in high-level API
+* [SAMZA-1392](https://issues.apache.org/jira/browse/SAMZA-1392) - KafkaSystemProducer performance and correctness with concurrent sends and flushes
+* [SAMZA-1406](https://issues.apache.org/jira/browse/SAMZA-1406) - Enhancements to the ZooKeeper-based deployment model
+* [SAMZA-1321](https://issues.apache.org/jira/browse/SAMZA-1321) - Support for multi-stage batch processing
 
 ## Upgrade Notes
 <!-- Add detailed notes on how someone using an older version of samza (typically, currentVersion - 1) can upgrade to the latest -->
@@ -71,5 +78,5 @@ If you have a custom **System** implementation, then you have to update to the n
 boolean clearStream(StreamSpec streamSpec);
 ```
 <br />
-Read more about it in the [API docs]().
+Read more about it in the [API docs](/learn/documentation/{{site.version}}/api/javadocs/org/apache/samza/system/SystemAdmin.html).
 

From d8fdb1a586a4379fcbecbc618a2090a508af63b0 Mon Sep 17 00:00:00 2001
From: Xinyu Liu <xinyuliu.us@gmail.com>
Date: Wed, 3 Jan 2018 16:10:45 -0800
Subject: [PATCH 25/36] SAMZA-1550: Update master to use 0.14.1-SNAPSHOT
 version

Update master to use 0.14.1-SNAPSHOT version.

Author: xiliu <xiliu@xiliu-ld1.linkedin.biz>

Reviewers: Yi Pan <nickpan47@gmail.com>

Closes #400 from xinyuiscool/SAMZA-1550-2
---
 docs/_config.yml                                              | 2 +-
 docs/learn/documentation/versioned/azure/eventhubs.md         | 2 +-
 docs/learn/tutorials/versioned/hello-samza-high-level-yarn.md | 2 +-
 docs/learn/tutorials/versioned/hello-samza-high-level-zk.md   | 2 +-
 docs/learn/tutorials/versioned/samza-rest-getting-started.md  | 2 +-
 docs/startup/hello-samza/versioned/index.md                   | 2 +-
 gradle.properties                                             | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/_config.yml b/docs/_config.yml
index 11850aa28f..cfd3b0eec9 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -25,4 +25,4 @@ exclude: [_docs]
 baseurl: http://samza.apache.org
 version: latest
 # this is the version you will go if you click 'switch version' in "latest" pages.
-latest-release: '0.13'
+latest-release: '0.14'
diff --git a/docs/learn/documentation/versioned/azure/eventhubs.md b/docs/learn/documentation/versioned/azure/eventhubs.md
index 2defddb9d2..7d76be35d1 100644
--- a/docs/learn/documentation/versioned/azure/eventhubs.md
+++ b/docs/learn/documentation/versioned/azure/eventhubs.md
@@ -165,7 +165,7 @@ With the environment setup complete, let us move on to building the hello-samza
 ```
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.14.0-SNAPSHOT-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.1-SNAPSHOT-dist.tar.gz -C deploy/samza
 ```
 
 We are now all set to deploy the application locally.
diff --git a/docs/learn/tutorials/versioned/hello-samza-high-level-yarn.md b/docs/learn/tutorials/versioned/hello-samza-high-level-yarn.md
index fbd1e05ef8..15e71272e4 100644
--- a/docs/learn/tutorials/versioned/hello-samza-high-level-yarn.md
+++ b/docs/learn/tutorials/versioned/hello-samza-high-level-yarn.md
@@ -63,7 +63,7 @@ Then, you can continue w/ the following command in hello-samza project:
 {% highlight bash %}
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.14.0-SNAPSHOT-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.1-SNAPSHOT-dist.tar.gz -C deploy/samza
 {% endhighlight %}
 
 ### Run a Samza Application
diff --git a/docs/learn/tutorials/versioned/hello-samza-high-level-zk.md b/docs/learn/tutorials/versioned/hello-samza-high-level-zk.md
index ebe5595e32..b2b1b4ea3f 100644
--- a/docs/learn/tutorials/versioned/hello-samza-high-level-zk.md
+++ b/docs/learn/tutorials/versioned/hello-samza-high-level-zk.md
@@ -59,7 +59,7 @@ With the environment setup complete, let us move on to building the hello-samza
 {% highlight bash %}
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.14.0-SNAPSHOT-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.1-SNAPSHOT-dist.tar.gz -C deploy/samza
 {% endhighlight %}
 
 We are now all set to deploy the application locally.
diff --git a/docs/learn/tutorials/versioned/samza-rest-getting-started.md b/docs/learn/tutorials/versioned/samza-rest-getting-started.md
index b889dd3754..54e3bdf32d 100644
--- a/docs/learn/tutorials/versioned/samza-rest-getting-started.md
+++ b/docs/learn/tutorials/versioned/samza-rest-getting-started.md
@@ -48,7 +48,7 @@ Run the following commands:
 {% highlight bash %}
 cd samza-rest/build/distributions/
 mkdir -p deploy/samza-rest
-tar -xvf ./samza-rest-0.14.0-SNAPSHOT.tgz -C deploy/samza-rest
+tar -xvf ./samza-rest-0.14.1-SNAPSHOT.tgz -C deploy/samza-rest
 {% endhighlight %}
 
 #### Configure the Installations Path
diff --git a/docs/startup/hello-samza/versioned/index.md b/docs/startup/hello-samza/versioned/index.md
index e94e2a5f40..f2461109cf 100644
--- a/docs/startup/hello-samza/versioned/index.md
+++ b/docs/startup/hello-samza/versioned/index.md
@@ -61,7 +61,7 @@ Then, you can continue w/ the following command in hello-samza project:
 {% highlight bash %}
 mvn clean package
 mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.14.0-SNAPSHOT-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.1-SNAPSHOT-dist.tar.gz -C deploy/samza
 {% endhighlight %}
 
 ### Run a Samza Job
diff --git a/gradle.properties b/gradle.properties
index f3cfad720e..bf37e1dc33 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 group=org.apache.samza
-version=0.13.1-SNAPSHOT
+version=0.14.1-SNAPSHOT
 scalaVersion=2.11
 
 gradleVersion=2.8

From 7954815ca0887f12fa11d4b4239eb5cd867b0711 Mon Sep 17 00:00:00 2001
From: xiliu <xiliu@xiliu-ld1.linkedin.biz>
Date: Thu, 4 Jan 2018 09:51:20 -0800
Subject: [PATCH 26/36] Fix a link in release-notes.md

---
 docs/startup/releases/versioned/release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/startup/releases/versioned/release-notes.md b/docs/startup/releases/versioned/release-notes.md
index f83978d291..18983f13b0 100644
--- a/docs/startup/releases/versioned/release-notes.md
+++ b/docs/startup/releases/versioned/release-notes.md
@@ -60,7 +60,7 @@ All Samza JARs are published through [Apache's Maven repository](https://reposit
 
 <!-- PR 290 -->
 * Introduced a new **mandatory** configuration - `job.coordination.utils.factory`. Read more about it
-[here](../../learn/{{site.version}}/configuration.html). <br />This config is applicable to all Samza
+[here](/learn/documentation/{{site.version}}/jobs/configuration-table.html). <br />This config is applicable to all Samza
 applications deployed using the `LocalApplicationRunner` (that is, non-yarn deployments).
 
 ### API Changes

From 5238aaa6cee81a87079c9d432204422ececea793 Mon Sep 17 00:00:00 2001
From: Xinyu Liu <xinyuliu.us@gmail.com>
Date: Tue, 9 Jan 2018 10:48:10 -0800
Subject: [PATCH 27/36] SAMZA-1553: Add log4j for latest Kafka build

Add it so Samza compiles with the latest kafka.

Author: xiliu <xiliu@xiliu-ld1.linkedin.biz>

Reviewers: Boris Shkolnik <sborya@gmail.com>

Closes #402 from xinyuiscool/SAMZA-1553
---
 build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build.gradle b/build.gradle
index 04b42f261d..c12b1777db 100644
--- a/build.gradle
+++ b/build.gradle
@@ -373,6 +373,7 @@ project(":samza-kafka_$scalaVersion") {
   dependencies {
     compile project(':samza-api')
     compile project(":samza-core_$scalaVersion")
+    compile "log4j:log4j:$log4jVersion"
     compile "org.scala-lang:scala-library:$scalaLibVersion"
     compile "com.101tec:zkclient:$zkClientVersion"
     compile "org.apache.zookeeper:zookeeper:$zookeeperVersion"

From b00ebd21a8a3cc064009509e5b953008c89a08f6 Mon Sep 17 00:00:00 2001
From: xiliu <xiliu@xiliu-ld1.linkedin.biz>
Date: Tue, 9 Jan 2018 15:53:40 -0800
Subject: [PATCH 28/36] SAMZA-1550: Update samza 0.14 version in tests

---
 samza-test/src/main/config/join/README        | 8 ++++----
 samza-test/src/main/python/configs/tests.json | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/samza-test/src/main/config/join/README b/samza-test/src/main/config/join/README
index 01a354b1b9..9392d3b5fe 100644
--- a/samza-test/src/main/config/join/README
+++ b/samza-test/src/main/config/join/README
@@ -44,17 +44,17 @@ Deploy Zookeeper, YARN and Kafka:
 > cd $HELLO_SAMZA_SRC
 > for i in zookeeper kafka yarn; do ./bin/grid install $i; ./bin/grid start $i; done
 
-Update the "yarn.package.path" to $DEPLOY_DIR/samza-test_2.11-0.13.1-SNAPSHOT.tgz
+Update the "yarn.package.path" to $DEPLOY_DIR/samza-test_2.11-0.14.1-SNAPSHOT.tgz
 > cd $SAMZA_SRC
 > vi samza-test/src/main/config/join/common.properties
-yarn.package.path=file:///path/to/samza-hello-samza/deploy/samza-test_2.11-0.13.1-SNAPSHOT.tgz
+yarn.package.path=file:///path/to/samza-hello-samza/deploy/samza-test_2.11-0.14.1-SNAPSHOT.tgz
 
 Then release and extract the test tarball:
 > cd $SAMZA_SRC
 > ./gradlew releaseTestJobs
-> cp samza-test/build/distributions/samza-test_2.11-0.13.1-SNAPSHOT.tgz $DEPLOY_DIR
+> cp samza-test/build/distributions/samza-test_2.11-0.14.1-SNAPSHOT.tgz $DEPLOY_DIR
 > mkdir $DEPLOY_DIR/samza
-> tar -xvf $DEPLOY_DIR/samza-test_2.11-0.13.1-SNAPSHOT.tgz -C $DEPLOY_DIR/samza
+> tar -xvf $DEPLOY_DIR/samza-test_2.11-0.14.1-SNAPSHOT.tgz -C $DEPLOY_DIR/samza
 
 Finally, create the kafka topics and start the samza jobs:
 > ./bin/setup-int-test.sh $DEPLOY_DIR
diff --git a/samza-test/src/main/python/configs/tests.json b/samza-test/src/main/python/configs/tests.json
index 0d37c301d8..3c72875ea6 100644
--- a/samza-test/src/main/python/configs/tests.json
+++ b/samza-test/src/main/python/configs/tests.json
@@ -1,5 +1,5 @@
 {
-  "samza_executable": "samza-test_2.11-0.13.1-SNAPSHOT.tgz",
+  "samza_executable": "samza-test_2.11-0.14.1-SNAPSHOT.tgz",
   "samza_install_path": "deploy/smoke_tests",
   "samza_config_factory": "org.apache.samza.config.factories.PropertiesConfigFactory"
 }

From 07f28e9483ba9c69f44fd7634f6fef6dc47ecd0c Mon Sep 17 00:00:00 2001
From: xiliu <xiliu@xiliu-ld1.linkedin.biz>
Date: Wed, 10 Jan 2018 10:33:23 -0800
Subject: [PATCH 29/36] Revert "SAMZA-1553: Add log4j for latest Kafka build"

This reverts commit 5238aaa6cee81a87079c9d432204422ececea793.
---
 build.gradle | 1 -
 1 file changed, 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index c12b1777db..04b42f261d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -373,7 +373,6 @@ project(":samza-kafka_$scalaVersion") {
   dependencies {
     compile project(':samza-api')
     compile project(":samza-core_$scalaVersion")
-    compile "log4j:log4j:$log4jVersion"
     compile "org.scala-lang:scala-library:$scalaLibVersion"
     compile "com.101tec:zkclient:$zkClientVersion"
     compile "org.apache.zookeeper:zookeeper:$zookeeperVersion"

From a6540b4e3d1d3916210c50be9b4b2b4920f885bb Mon Sep 17 00:00:00 2001
From: Dong Lin <lindong28@gmail.com>
Date: Wed, 10 Jan 2018 10:52:38 -0800
Subject: [PATCH 30/36] SAMZA-1530; Bump up Kafka dependency to 0.11

Author: Dong Lin <lindong28@gmail.com>

Reviewers: Xinyu Liu <xinyuliu.us@gmail.com>

Closes #395 from lindong28/SAMZA-1530
---
 bin/check-all.sh                              |  2 +-
 gradle/dependency-versions.gradle             |  2 +-
 .../samza/system/kafka/BrokerProxy.scala      | 13 ++++----
 .../apache/samza/system/kafka/GetOffset.scala |  4 +--
 .../samza/system/kafka/KafkaSystemAdmin.scala |  8 ++---
 .../system/kafka/TopicMetadataCache.scala     |  2 +-
 .../org/apache/samza/util/KafkaUtil.scala     |  8 +++--
 .../samza/system/kafka/MockKafkaProducer.java | 25 +++++++++++++--
 .../kafka/TestKafkaCheckpointManager.scala    |  7 ++--
 .../samza/system/kafka/TestBrokerProxy.scala  |  9 +++---
 .../system/kafka/TestKafkaSystemAdmin.scala   | 26 ++++++---------
 .../kafka/TestKafkaSystemConsumer.scala       |  4 +--
 .../system/kafka/TestTopicMetadataCache.scala | 32 ++++++++++---------
 .../apache/samza/utils/TestKafkaUtil.scala    |  7 ++--
 .../TestZkLocalApplicationRunner.java         |  2 +-
 .../test/integration/StreamTaskTestUtil.scala |  7 ++--
 16 files changed, 86 insertions(+), 72 deletions(-)

diff --git a/bin/check-all.sh b/bin/check-all.sh
index 2f9f03c62a..f168bc8d15 100755
--- a/bin/check-all.sh
+++ b/bin/check-all.sh
@@ -21,7 +21,7 @@
 
 set -e
 
-SCALAs=( "2.10" "2.11" "2.12" )
+SCALAs=( "2.11" "2.12" )
 JDKs=( "JAVA8_HOME" )
 YARNs=( "2.6.1" "2.7.1" )
 
diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle
index 20a1d56efd..2e45914f59 100644
--- a/gradle/dependency-versions.gradle
+++ b/gradle/dependency-versions.gradle
@@ -33,7 +33,7 @@
   jodaTimeVersion = "2.2"
   joptSimpleVersion = "3.2"
   junitVersion = "4.8.1"
-  kafkaVersion = "0.10.1.1"
+  kafkaVersion = "0.11.0.2"
   log4jVersion = "1.2.17"
   metricsVersion = "2.2.0"
   mockitoVersion = "1.10.19"
diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/BrokerProxy.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/BrokerProxy.scala
index 5338886254..8a6618d238 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/BrokerProxy.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/BrokerProxy.scala
@@ -182,7 +182,7 @@ class BrokerProxy(
       firstCallBarrier.countDown()
 
       // Split response into errors and non errors, processing the errors first
-      val (nonErrorResponses, errorResponses) = response.data.toSet.partition(_._2.error == ErrorMapping.NoError)
+      val (nonErrorResponses, errorResponses) = response.data.toSet.partition(_._2.error.code() == ErrorMapping.NoError)
 
       handleErrors(errorResponses, response)
 
@@ -219,18 +219,17 @@ class BrokerProxy(
     immutableNextOffsetsCopy.keySet.foreach(abdicate(_))
   }
 
-  def handleErrors(errorResponses: Set[(TopicAndPartition, FetchResponsePartitionData)], response:FetchResponse) = {
+  def handleErrors(errorResponses: Set[(TopicAndPartition, FetchResponsePartitionData)], response: FetchResponse) = {
     // FetchResponse should really return Option and a list of the errors so we don't have to find them ourselves
-    case class Error(tp: TopicAndPartition, code: Short, exception: Throwable)
+    case class Error(tp: TopicAndPartition, code: Short, exception: Exception)
 
     // Now subdivide the errors into three types: non-recoverable, not leader (== abdicate) and offset out of range (== get new offset)
 
     // Convert FetchResponse into easier-to-work-with Errors
     val errors = for (
       (topicAndPartition, responseData) <- errorResponses;
-      errorCode <- Option(response.errorCode(topicAndPartition.topic, topicAndPartition.partition)); // Scala's being cranky about referring to error.getKey values...
-      exception <- Option(ErrorMapping.exceptionFor(errorCode))
-    ) yield new Error(topicAndPartition, errorCode, exception)
+      error <- Option(response.error(topicAndPartition.topic, topicAndPartition.partition)) // Scala's being cranky about referring to error.getKey values...
+    ) yield new Error(topicAndPartition, error.code(), error.exception())
 
     val (notLeaderOrUnknownTopic, otherErrors) = errors.partition { case (e) => e.code == ErrorMapping.NotLeaderForPartitionCode || e.code == ErrorMapping.UnknownTopicOrPartitionCode }
     val (offsetOutOfRangeErrors, remainingErrors) = otherErrors.partition(_.code == ErrorMapping.OffsetOutOfRangeCode)
@@ -241,7 +240,7 @@ class BrokerProxy(
     // handle the recoverable errors.
     remainingErrors.foreach(e => {
       warn("Got non-recoverable error codes during multifetch. Throwing an exception to trigger reconnect. Errors: %s" format remainingErrors.mkString(","))
-      KafkaUtil.maybeThrowException(e.code) })
+      KafkaUtil.maybeThrowException(e.exception) })
 
     notLeaderOrUnknownTopic.foreach(e => abdicate(e.tp))
 
diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/GetOffset.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/GetOffset.scala
index 5528702070..040e2460be 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/GetOffset.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/GetOffset.scala
@@ -60,7 +60,7 @@ class GetOffset(
       val messages = consumer.defaultFetch((topicAndPartition, offset.toLong))
 
       if (messages.hasError) {
-        KafkaUtil.maybeThrowException(messages.errorCode(topicAndPartition.topic, topicAndPartition.partition))
+        KafkaUtil.maybeThrowException(messages.error(topicAndPartition.topic, topicAndPartition.partition).exception())
       }
 
       info("Able to successfully read from offset %s for topic and partition %s. Using it to instantiate consumer." format (offset, topicAndPartition))
@@ -86,7 +86,7 @@ class GetOffset(
       .get(topicAndPartition)
       .getOrElse(toss("Unable to find offset information for %s" format topicAndPartition))
 
-    KafkaUtil.maybeThrowException(partitionOffsetResponse.error)
+    KafkaUtil.maybeThrowException(partitionOffsetResponse.error.exception())
 
     partitionOffsetResponse
       .offsets
diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdmin.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdmin.scala
index 013b292d0a..4715141267 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdmin.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdmin.scala
@@ -164,7 +164,7 @@ class KafkaSystemAdmin(
           metadataTTL)
         val result = metadata.map {
           case (topic, topicMetadata) => {
-            KafkaUtil.maybeThrowException(topicMetadata.errorCode)
+            KafkaUtil.maybeThrowException(topicMetadata.error.exception())
             val partitionsMap = topicMetadata.partitionsMetadata.map {
               pm =>
                 new Partition(pm.partitionId) -> new SystemStreamPartitionMetadata("", "", "")
@@ -350,7 +350,7 @@ class KafkaSystemAdmin(
       .values
       // Convert the topic metadata to a Seq[(Broker, TopicAndPartition)]
       .flatMap(topicMetadata => {
-        KafkaUtil.maybeThrowException(topicMetadata.errorCode)
+        KafkaUtil.maybeThrowException(topicMetadata.error.exception())
         topicMetadata
           .partitionsMetadata
           // Convert Seq[PartitionMetadata] to Seq[(Broker, TopicAndPartition)]
@@ -390,7 +390,7 @@ class KafkaSystemAdmin(
       .getOffsetsBefore(new OffsetRequest(partitionOffsetInfo))
       .partitionErrorAndOffsets
       .mapValues(partitionErrorAndOffset => {
-        KafkaUtil.maybeThrowException(partitionErrorAndOffset.error)
+        KafkaUtil.maybeThrowException(partitionErrorAndOffset.error.exception())
         partitionErrorAndOffset.offsets.head
       })
 
@@ -480,7 +480,7 @@ class KafkaSystemAdmin(
         val metadataStore = new ClientUtilTopicMetadataStore(brokerListString, clientId, timeout)
         val topicMetadataMap = TopicMetadataCache.getTopicMetadata(Set(topicName), systemName, metadataStore.getTopicInfo, metadataTTL)
         val topicMetadata = topicMetadataMap(topicName)
-        KafkaUtil.maybeThrowException(topicMetadata.errorCode)
+        KafkaUtil.maybeThrowException(topicMetadata.error.exception())
 
         val partitionCount = topicMetadata.partitionsMetadata.length
         if (partitionCount != spec.getPartitionCount) {
diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/TopicMetadataCache.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/TopicMetadataCache.scala
index 82ecf1aee9..8a3ab2bc62 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/TopicMetadataCache.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/TopicMetadataCache.scala
@@ -73,6 +73,6 @@ object TopicMetadataCache extends Logging {
    * partition's metadata has a bad errorCode.
    */
   def hasBadErrorCode(streamMetadata: TopicMetadata) = {
-    KafkaUtil.isBadErrorCode(streamMetadata.errorCode) || streamMetadata.partitionsMetadata.exists(partitionMetadata => KafkaUtil.isBadErrorCode(partitionMetadata.errorCode))
+    KafkaUtil.isBadErrorCode(streamMetadata.error.code()) || streamMetadata.partitionsMetadata.exists(partitionMetadata => KafkaUtil.isBadErrorCode(partitionMetadata.error.code()))
   }
 }
diff --git a/samza-kafka/src/main/scala/org/apache/samza/util/KafkaUtil.scala b/samza-kafka/src/main/scala/org/apache/samza/util/KafkaUtil.scala
index 1410cbbce0..5b0137a1ee 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/util/KafkaUtil.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/util/KafkaUtil.scala
@@ -29,7 +29,8 @@ import org.apache.samza.config.{ApplicationConfig, Config, ConfigException}
 import org.apache.samza.config.JobConfig.Config2Job
 import org.apache.samza.execution.StreamManager
 import org.apache.samza.system.OutgoingMessageEnvelope
-import kafka.common.{ErrorMapping, ReplicaNotAvailableException}
+import org.apache.kafka.common.errors.ReplicaNotAvailableException
+import kafka.common.ErrorMapping
 import org.apache.kafka.common.errors.TopicExistsException
 import org.apache.samza.system.kafka.TopicMetadataCache
 
@@ -71,9 +72,10 @@ object KafkaUtil extends Logging {
    * <a href="https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol">protocol
    * docs</a>, ReplicaNotAvailableException can be safely ignored.
    */
-  def maybeThrowException(code: Short) {
+  def maybeThrowException(e: Exception) {
     try {
-      ErrorMapping.maybeThrowException(code)
+      if (e != null)
+        throw e
     } catch {
       case e: ReplicaNotAvailableException =>
         debug("Got ReplicaNotAvailableException, but ignoring since it's safe to do so.")
diff --git a/samza-kafka/src/test/java/org/apache/samza/system/kafka/MockKafkaProducer.java b/samza-kafka/src/test/java/org/apache/samza/system/kafka/MockKafkaProducer.java
index 024c6e6330..e66b7c3198 100644
--- a/samza-kafka/src/test/java/org/apache/samza/system/kafka/MockKafkaProducer.java
+++ b/samza-kafka/src/test/java/org/apache/samza/system/kafka/MockKafkaProducer.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import kafka.producer.ProducerClosedException;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
 import org.apache.kafka.clients.producer.Callback;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
@@ -42,6 +43,7 @@
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.record.Record;
+import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.test.TestUtils;
 
 public class MockKafkaProducer implements Producer<byte[], byte[]> {
@@ -98,7 +100,7 @@ public Future<RecordMetadata> send(ProducerRecord record) {
   }
 
   private RecordMetadata getRecordMetadata(ProducerRecord record) {
-    return new RecordMetadata(new TopicPartition(record.topic(), record.partition() == null ? 0 : record.partition()), 0, this.msgsSent.get(), Record.NO_TIMESTAMP, -1, -1, -1);
+    return new RecordMetadata(new TopicPartition(record.topic(), record.partition() == null ? 0 : record.partition()), 0, this.msgsSent.get(), -1L, -1, -1, -1);
   }
 
   @Override
@@ -190,6 +192,25 @@ public synchronized void flush () {
     new FlushRunnable(0).run();
   }
 
+  public void initTransactions() {
+
+  }
+
+  public void abortTransaction() {
+
+  }
+
+  public void beginTransaction() {
+
+  }
+
+  public void commitTransaction() {
+
+  }
+
+  public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId) {
+    
+  }
 
   private static class FutureFailure implements Future<RecordMetadata> {
 
@@ -232,7 +253,7 @@ private static class FutureSuccess implements Future<RecordMetadata> {
 
     public FutureSuccess(ProducerRecord record, int offset) {
       this.record = record;
-      this._metadata = new RecordMetadata(new TopicPartition(record.topic(), record.partition() == null ? 0 : record.partition()), 0, offset, Record.NO_TIMESTAMP, -1, -1, -1);
+      this._metadata = new RecordMetadata(new TopicPartition(record.topic(), record.partition() == null ? 0 : record.partition()), 0, offset, RecordBatch.NO_TIMESTAMP, -1, -1, -1);
     }
 
     @Override
diff --git a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala
index ec9f3a0952..86cb4185f0 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala
@@ -52,15 +52,12 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness {
   val checkpoint1 = new Checkpoint(ImmutableMap.of(ssp, "offset-1"))
   val checkpoint2 = new Checkpoint(ImmutableMap.of(ssp, "offset-2"))
   val taskName = new TaskName("Partition 0")
-
-  var brokers: String = null
   var config: Config = null
 
   @Before
   override def setUp {
     super.setUp
     TestUtils.waitUntilTrue(() => servers.head.metadataCache.getAliveBrokers.size == numBrokers, "Wait for cache to update")
-    brokers = brokerList.split(",").map(p => "localhost" + p).mkString(",")
     config = getConfig()
   }
 
@@ -140,7 +137,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness {
     val defaultSerializer = classOf[ByteArraySerializer].getCanonicalName
     val props = new Properties()
     props.putAll(ImmutableMap.of(
-      ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokers,
+      ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList,
       ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, defaultSerializer,
       ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, defaultSerializer))
     props
@@ -151,7 +148,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness {
       .put(JobConfig.JOB_NAME, "some-job-name")
       .put(JobConfig.JOB_ID, "i001")
       .put(s"systems.$checkpointSystemName.samza.factory", classOf[KafkaSystemFactory].getCanonicalName)
-      .put(s"systems.$checkpointSystemName.producer.bootstrap.servers", brokers)
+      .put(s"systems.$checkpointSystemName.producer.bootstrap.servers", brokerList)
       .put(s"systems.$checkpointSystemName.consumer.zookeeper.connect", zkConnect)
       .put("task.checkpoint.system", checkpointSystemName)
       .build())
diff --git a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestBrokerProxy.scala b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestBrokerProxy.scala
index f0bdafdc21..d510076792 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestBrokerProxy.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestBrokerProxy.scala
@@ -24,9 +24,10 @@ import java.nio.ByteBuffer
 import java.util.concurrent.CountDownLatch
 
 import kafka.api.{PartitionOffsetsResponse, _}
-import kafka.common.{ErrorMapping, TopicAndPartition}
+import kafka.common.TopicAndPartition
 import kafka.consumer.SimpleConsumer
 import kafka.message.{ByteBufferMessageSet, Message, MessageAndOffset, MessageSet}
+import org.apache.kafka.common.protocol.Errors
 import org.apache.samza.SamzaException
 import org.apache.samza.util.Logging
 import org.junit.Assert._
@@ -165,7 +166,7 @@ class TestBrokerProxy extends Logging {
               messageSet
             }
 
-            val fetchResponsePartitionData = FetchResponsePartitionData(0, 500, messageSet)
+            val fetchResponsePartitionData = FetchResponsePartitionData(Errors.NONE, 500, messageSet)
             val map = scala.Predef.Map[TopicAndPartition, FetchResponsePartitionData](tp -> fetchResponsePartitionData)
 
             when(fetchResponse.data).thenReturn(map.toSeq)
@@ -257,12 +258,12 @@ class TestBrokerProxy extends Logging {
           }
           val mfr = mock(classOf[FetchResponse])
           when(mfr.hasError).thenReturn(true)
-          when(mfr.errorCode("topic", 42)).thenReturn(ErrorMapping.OffsetOutOfRangeCode)
+          when(mfr.error("topic", 42)).thenReturn(Errors.OFFSET_OUT_OF_RANGE)
 
           val messageSet = mock(classOf[MessageSet])
           when(messageSet.iterator).thenReturn(Iterator.empty)
           val response = mock(classOf[FetchResponsePartitionData])
-          when(response.error).thenReturn(ErrorMapping.OffsetOutOfRangeCode)
+          when(response.error).thenReturn(Errors.OFFSET_OUT_OF_RANGE)
           val responseMap = Map(tp -> response)
           when(mfr.data).thenReturn(responseMap.toSeq)
           invocationCount += 1
diff --git a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala
index 65c43f5cf9..2039447947 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala
@@ -24,7 +24,8 @@ package org.apache.samza.system.kafka
 import java.util.{Properties, UUID}
 
 import kafka.admin.AdminUtils
-import kafka.common.{ErrorMapping, LeaderNotAvailableException}
+import org.apache.kafka.common.errors.LeaderNotAvailableException
+import org.apache.kafka.common.protocol.Errors
 import kafka.consumer.{Consumer, ConsumerConfig}
 import kafka.integration.KafkaServerTestHarness
 import kafka.server.KafkaConfig
@@ -68,19 +69,13 @@ object TestKafkaSystemAdmin extends KafkaServerTestHarness {
   @BeforeClass
   override def setUp {
     super.setUp
-
     val config = new java.util.HashMap[String, String]()
-
-    brokers = brokerList.split(",").map(p => "localhost" + p).mkString(",")
-
-    config.put("bootstrap.servers", brokers)
+    config.put("bootstrap.servers", brokerList)
     config.put("acks", "all")
     config.put("serializer.class", "kafka.serializer.StringEncoder")
-
     producerConfig = new KafkaProducerConfig("kafka", "i001", config)
-
     producer = new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties)
-    metadataStore = new ClientUtilTopicMetadataStore(brokers, "some-job-name")
+    metadataStore = new ClientUtilTopicMetadataStore(brokerList, "some-job-name")
   }
 
 
@@ -107,9 +102,8 @@ object TestKafkaSystemAdmin extends KafkaServerTestHarness {
       try {
         val topicMetadataMap = TopicMetadataCache.getTopicMetadata(Set(topic), SYSTEM, metadataStore.getTopicInfo)
         val topicMetadata = topicMetadataMap(topic)
-        val errorCode = topicMetadata.errorCode
 
-        KafkaUtil.maybeThrowException(errorCode)
+        KafkaUtil.maybeThrowException(topicMetadata.error.exception())
 
         done = expectedPartitionCount == topicMetadata.partitionsMetadata.size
       } catch {
@@ -137,11 +131,11 @@ object TestKafkaSystemAdmin extends KafkaServerTestHarness {
   }
 
   def createSystemAdmin: KafkaSystemAdmin = {
-    new KafkaSystemAdmin(SYSTEM, brokers, connectZk = () => ZkUtils(zkConnect, 6000, 6000, zkSecure))
+    new KafkaSystemAdmin(SYSTEM, brokerList, connectZk = () => ZkUtils(zkConnect, 6000, 6000, zkSecure))
   }
 
   def createSystemAdmin(coordinatorStreamProperties: Properties, coordinatorStreamReplicationFactor: Int, topicMetaInformation: Map[String, ChangelogInfo]): KafkaSystemAdmin = {
-    new KafkaSystemAdmin(SYSTEM, brokers, connectZk = () => ZkUtils(zkConnect, 6000, 6000, zkSecure), coordinatorStreamProperties, coordinatorStreamReplicationFactor, 10000, ConsumerConfig.SocketBufferSize, UUID.randomUUID.toString, topicMetaInformation, Map())
+    new KafkaSystemAdmin(SYSTEM, brokerList, connectZk = () => ZkUtils(zkConnect, 6000, 6000, zkSecure), coordinatorStreamProperties, coordinatorStreamReplicationFactor, 10000, ConsumerConfig.SocketBufferSize, UUID.randomUUID.toString, topicMetaInformation, Map())
   }
 
 }
@@ -281,7 +275,7 @@ class TestKafkaSystemAdmin {
   @Test
   def testShouldCreateCoordinatorStream {
     val topic = "test-coordinator-stream"
-    val systemAdmin = new KafkaSystemAdmin(SYSTEM, brokers, () => ZkUtils(zkConnect, 6000, 6000, zkSecure), coordinatorStreamReplicationFactor = 3)
+    val systemAdmin = new KafkaSystemAdmin(SYSTEM, brokerList, () => ZkUtils(zkConnect, 6000, 6000, zkSecure), coordinatorStreamReplicationFactor = 3)
 
     val spec = StreamSpec.createCoordinatorStreamSpec(topic, "kafka")
     systemAdmin.createStream(spec)
@@ -294,14 +288,14 @@ class TestKafkaSystemAdmin {
     assertEquals(3, partitionMetadata.replicas.size)
   }
 
-  class KafkaSystemAdminWithTopicMetadataError extends KafkaSystemAdmin(SYSTEM, brokers, () => ZkUtils(zkConnect, 6000, 6000, zkSecure)) {
+  class KafkaSystemAdminWithTopicMetadataError extends KafkaSystemAdmin(SYSTEM, brokerList, () => ZkUtils(zkConnect, 6000, 6000, zkSecure)) {
     import kafka.api.TopicMetadata
     var metadataCallCount = 0
 
     // Simulate Kafka telling us that the leader for the topic is not available
     override def getTopicMetadata(topics: Set[String]) = {
       metadataCallCount += 1
-      val topicMetadata = TopicMetadata(topic = "quux", partitionsMetadata = Seq(), errorCode = ErrorMapping.LeaderNotAvailableCode)
+      val topicMetadata = TopicMetadata(topic = "quux", partitionsMetadata = Seq(), error = Errors.LEADER_NOT_AVAILABLE)
       Map("quux" -> topicMetadata)
     }
   }
diff --git a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemConsumer.scala b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemConsumer.scala
index 8a5cbc2581..4dd170fc4d 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemConsumer.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemConsumer.scala
@@ -25,7 +25,7 @@ import kafka.cluster.Broker
 import kafka.common.TopicAndPartition
 import kafka.message.Message
 import kafka.message.MessageAndOffset
-
+import org.apache.kafka.common.protocol.Errors
 import org.apache.samza.system.IncomingMessageEnvelope
 import org.apache.samza.system.SystemStreamPartition
 import org.apache.samza.Partition
@@ -68,7 +68,7 @@ class TestKafkaSystemConsumer {
     // Lie and tell the store that the partition metadata is empty. We can't
     // use partition metadata because it has Broker in its constructor, which
     // is package private to Kafka.
-    val metadataStore = new MockMetadataStore(Map(streamName -> TopicMetadata(streamName, Seq.empty, 0)))
+    val metadataStore = new MockMetadataStore(Map(streamName -> TopicMetadata(streamName, Seq.empty, Errors.NONE)))
     var hosts = List[String]()
     var getHostPortCount = 0
     val consumer = new KafkaSystemConsumer(systemName, systemAdmin, metrics, metadataStore, clientId) {
diff --git a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestTopicMetadataCache.scala b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestTopicMetadataCache.scala
index 50c13ababb..9cc2f63bbe 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestTopicMetadataCache.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestTopicMetadataCache.scala
@@ -20,7 +20,8 @@
 package org.apache.samza.system.kafka
 
 import java.util.concurrent.CountDownLatch
-import java.util.concurrent.atomic.{ AtomicBoolean, AtomicInteger }
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
+
 import kafka.api.TopicMetadata
 import org.I0Itec.zkclient.ZkClient
 import org.apache.samza.util.Clock
@@ -30,6 +31,7 @@ import org.junit.Before
 import org.junit.Test
 import kafka.common.ErrorMapping
 import kafka.api.PartitionMetadata
+import org.apache.kafka.common.protocol.Errors
 
 class TestTopicMetadataCache {
 
@@ -41,8 +43,8 @@ class TestTopicMetadataCache {
 
   class MockTopicMetadataStore extends TopicMetadataStore {
     var mockCache = Map(
-      "topic1" -> new TopicMetadata("topic1", List.empty, 0),
-      "topic2" -> new TopicMetadata("topic2", List.empty, 0))
+      "topic1" -> new TopicMetadata("topic1", List.empty, Errors.NONE),
+      "topic2" -> new TopicMetadata("topic2", List.empty, Errors.NONE))
     var numberOfCalls: AtomicInteger = new AtomicInteger(0)
 
     def getTopicInfo(topics: Set[String]) = {
@@ -53,7 +55,7 @@ class TestTopicMetadataCache {
     }
 
     def setErrorCode(topic: String, errorCode: Short) {
-      mockCache += topic -> new TopicMetadata(topic, List.empty, errorCode)
+      mockCache += topic -> new TopicMetadata(topic, List.empty, Errors.forCode(errorCode))
     }
   }
 
@@ -70,7 +72,7 @@ class TestTopicMetadataCache {
     mockStore.setErrorCode("topic1", 3)
     var metadata = TopicMetadataCache.getTopicMetadata(Set("topic1"), "kafka", mockStore.getTopicInfo, 5, mockTime.currentTimeMillis)
     assertEquals("topic1", metadata("topic1").topic)
-    assertEquals(3, metadata("topic1").errorCode)
+    assertEquals(3, metadata("topic1").error.code)
     assertEquals(1, mockStore.numberOfCalls.get())
 
     // Retrieve the same topic from the cache which has an error code. Ensure the store is called to refresh the cache
@@ -78,21 +80,21 @@ class TestTopicMetadataCache {
     mockStore.setErrorCode("topic1", 0)
     metadata = TopicMetadataCache.getTopicMetadata(Set("topic1"), "kafka", mockStore.getTopicInfo, 5, mockTime.currentTimeMillis)
     assertEquals("topic1", metadata("topic1").topic)
-    assertEquals(0, metadata("topic1").errorCode)
+    assertEquals(0, metadata("topic1").error.code)
     assertEquals(2, mockStore.numberOfCalls.get())
 
     // Retrieve the same topic from the cache with refresh rate greater than the last update. Ensure the store is not
     // called
     metadata = TopicMetadataCache.getTopicMetadata(Set("topic1"), "kafka", mockStore.getTopicInfo, 5, mockTime.currentTimeMillis)
     assertEquals("topic1", metadata("topic1").topic)
-    assertEquals(0, metadata("topic1").errorCode)
+    assertEquals(0, metadata("topic1").error.code)
     assertEquals(2, mockStore.numberOfCalls.get())
 
     // Ensure that refresh happens when refresh rate is less than the last update. Ensure the store is called
     mockTime.currentValue = 11
     metadata = TopicMetadataCache.getTopicMetadata(Set("topic1"), "kafka", mockStore.getTopicInfo, 5, mockTime.currentTimeMillis)
     assertEquals("topic1", metadata("topic1").topic)
-    assertEquals(0, metadata("topic1").errorCode)
+    assertEquals(0, metadata("topic1").error.code)
     assertEquals(3, mockStore.numberOfCalls.get())
   }
 
@@ -113,7 +115,7 @@ class TestTopicMetadataCache {
           waitForThreadStart.await()
           val metadata = TopicMetadataCache.getTopicMetadata(Set("topic1"), "kafka", mockStore.getTopicInfo, 5, mockTime.currentTimeMillis)
           numAssertionSuccess.compareAndSet(true, metadata("topic1").topic.equals("topic1"))
-          numAssertionSuccess.compareAndSet(true, metadata("topic1").errorCode == 0)
+          numAssertionSuccess.compareAndSet(true, metadata("topic1").error.code == 0)
         }
       })
       threads(i).start()
@@ -127,11 +129,11 @@ class TestTopicMetadataCache {
 
   @Test
   def testBadErrorCodes {
-    val partitionMetadataBad = new PartitionMetadata(0, None, Seq(), errorCode = ErrorMapping.LeaderNotAvailableCode)
-    val partitionMetadataGood = new PartitionMetadata(0, None, Seq(), errorCode = ErrorMapping.NoError)
-    assertTrue(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List.empty, ErrorMapping.RequestTimedOutCode)))
-    assertTrue(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List(partitionMetadataBad), ErrorMapping.NoError)))
-    assertFalse(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List.empty, ErrorMapping.NoError)))
-    assertFalse(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List(partitionMetadataGood), ErrorMapping.NoError)))
+    val partitionMetadataBad = new PartitionMetadata(0, None, Seq(), error = Errors.LEADER_NOT_AVAILABLE)
+    val partitionMetadataGood = new PartitionMetadata(0, None, Seq(), error = Errors.NONE)
+    assertTrue(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List.empty, Errors.REQUEST_TIMED_OUT)))
+    assertTrue(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List(partitionMetadataBad), Errors.NONE)))
+    assertFalse(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List.empty, Errors.NONE)))
+    assertFalse(TopicMetadataCache.hasBadErrorCode(new TopicMetadata("test", List(partitionMetadataGood), Errors.NONE)))
   }
 }
diff --git a/samza-kafka/src/test/scala/org/apache/samza/utils/TestKafkaUtil.scala b/samza-kafka/src/test/scala/org/apache/samza/utils/TestKafkaUtil.scala
index 848cfc8fcc..3548aea1a9 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/utils/TestKafkaUtil.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/utils/TestKafkaUtil.scala
@@ -20,14 +20,15 @@
 package org.apache.samza.utils
 
 import org.junit.Test
-import org.scalatest.{ Matchers => ScalaTestMatchers }
+import org.scalatest.{Matchers => ScalaTestMatchers}
 import org.apache.samza.util.KafkaUtil
 import kafka.common.ErrorMapping
+import org.apache.kafka.common.protocol.Errors
 
 class TestKafkaUtil extends ScalaTestMatchers {
   @Test
   def testMaybeThrowException {
-    intercept[Exception] { KafkaUtil.maybeThrowException(ErrorMapping.UnknownTopicOrPartitionCode) }
-    KafkaUtil.maybeThrowException(ErrorMapping.ReplicaNotAvailableCode)
+    intercept[Exception] { KafkaUtil.maybeThrowException(Errors.UNKNOWN_TOPIC_OR_PARTITION.exception()) }
+    KafkaUtil.maybeThrowException(Errors.REPLICA_NOT_AVAILABLE.exception())
   }
 }
\ No newline at end of file
diff --git a/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java b/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java
index 9c5dad5f54..97fe1f8532 100644
--- a/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java
+++ b/samza-test/src/test/java/org/apache/samza/test/processor/TestZkLocalApplicationRunner.java
@@ -541,7 +541,7 @@ public void shouldKillStreamAppWhenZooKeeperDiesBeforeLeaderReElection() throws
     // Set up kafka topics.
     publishKafkaEvents(inputKafkaTopic, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0]);
 
-    MapConfig kafkaProducerConfig = new MapConfig(ImmutableMap.of(String.format("systems.%s.producer.%s", TEST_SYSTEM, ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG), "1000"));
+    MapConfig kafkaProducerConfig = new MapConfig(ImmutableMap.of(String.format("systems.%s.producer.%s", TEST_SYSTEM, ProducerConfig.MAX_BLOCK_MS_CONFIG), "1000"));
     MapConfig applicationRunnerConfig1 = new MapConfig(ImmutableList.of(applicationConfig1, kafkaProducerConfig));
     MapConfig applicationRunnerConfig2 = new MapConfig(ImmutableList.of(applicationConfig2, kafkaProducerConfig));
     LocalApplicationRunner applicationRunner1 = new LocalApplicationRunner(applicationRunnerConfig1);
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
index 4ba51f34a3..d9261ad902 100644
--- a/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
@@ -117,9 +117,7 @@ object StreamTaskTestUtil {
     })
 
     servers = configs.map(TestUtils.createServer(_)).toBuffer
-
-    val brokerList = TestUtils.getBrokerListStrFromServers(servers, SecurityProtocol.PLAINTEXT)
-    brokers = brokerList.split(",").map(p => "127.0.0.1" + p).mkString(",")
+    brokers = TestUtils.getBrokerListStrFromServers(servers, SecurityProtocol.PLAINTEXT)
 
     // setup the zookeeper and bootstrap servers for local kafka cluster
     jobConfig ++= Map("systems.kafka.consumer.zookeeper.connect" -> zkConnect,
@@ -161,9 +159,8 @@ object StreamTaskTestUtil {
 
         topics.foreach(topic => {
           val topicMetadata = topicMetadataMap(topic)
-          val errorCode = topicMetadata.errorCode
 
-          KafkaUtil.maybeThrowException(errorCode)
+          KafkaUtil.maybeThrowException(topicMetadata.error.exception())
         })
 
         done = true

From 9893069cb5186d58e3cf5518ce8e5a7251d8b131 Mon Sep 17 00:00:00 2001
From: Srinivasulu Punuru <spunuru@linkedin.com>
Date: Wed, 10 Jan 2018 11:17:39 -0800
Subject: [PATCH 31/36] SAMZA-1535: Support for UDFs in where clauses

The existing version of the udf implementation doesn't seem to support udfs in the where clauses because the Type of the object returned is "ANY" and when you do a
`select * from kafka.topic where regexMatch('.*foo', Name)` it fails in the query validation, because calcite doesn't know the type of regexMatch.

To solve the problem, We made the scalarUdf generic with a strongly typed return type.

https://issues.apache.org/jira/browse/SAMZA-1535

This PR can be merged into trunk not the 0.14.

Author: Srinivasulu Punuru <spunuru@linkedin.com>

Reviewers: Xinyu Liu <xinyuliu.us@gmail.com>

Closes #386 from srinipunuru/udf-where.1
---
 .../org/apache/samza/sql/udfs/ScalarUdf.java     |  4 ++--
 .../apache/samza/sql/avro/AvroRelConverter.java  |  2 +-
 .../samza/sql/data/SamzaSqlRelMessage.java       |  2 ++
 .../java/org/apache/samza/sql/fn/FlattenUdf.java |  2 +-
 .../org/apache/samza/sql/fn}/RegexMatchUdf.java  |  7 +++----
 .../samza/sql/impl/ConfigBasedUdfResolver.java   | 12 ++++++------
 .../sql/planner/SamzaSqlScalarFunctionImpl.java  |  7 +++++--
 .../samza/sql/avro/TestAvroRelConversion.java    | 14 ++++++++++++++
 .../samza/sql/e2e/TestSamzaSqlEndToEnd.java      | 16 ++++++++++++++++
 .../samza/sql/testutil/MyTestArrayUdf.java       |  5 +++--
 .../org/apache/samza/sql/testutil/MyTestUdf.java |  2 +-
 .../samza/sql/testutil/SamzaSqlTestConfig.java   |  6 ++++--
 .../org/apache/samza/tools/SamzaSqlConsole.java  | 12 ++++++------
 13 files changed, 64 insertions(+), 27 deletions(-)
 rename {samza-tools/src/main/java/org/apache/samza/tools/udf => samza-sql/src/main/java/org/apache/samza/sql/fn}/RegexMatchUdf.java (88%)

diff --git a/samza-api/src/main/java/org/apache/samza/sql/udfs/ScalarUdf.java b/samza-api/src/main/java/org/apache/samza/sql/udfs/ScalarUdf.java
index 719cace87c..f5c111a42b 100644
--- a/samza-api/src/main/java/org/apache/samza/sql/udfs/ScalarUdf.java
+++ b/samza-api/src/main/java/org/apache/samza/sql/udfs/ScalarUdf.java
@@ -30,7 +30,7 @@
  *     select myudf(id, name) from profile
  * In the above query, Profile should contain fields named 'id' of INTEGER/NUMBER type and 'name' of type VARCHAR/CHARACTER
  */
-public interface ScalarUdf {
+public interface ScalarUdf<T> {
   /**
    * Udfs can implement this method to perform any initialization that they may need.
    * @param udfConfig Config specific to the udf.
@@ -44,5 +44,5 @@ public interface ScalarUdf {
    * @return
    *   Return value from the scalar udf.
    */
-  Object execute(Object... args);
+  T execute(Object... args);
 }
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java b/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
index ab46a9870c..086bb9362b 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/avro/AvroRelConverter.java
@@ -155,7 +155,7 @@ public SamzaSqlRelMessage convertToRelMessage(KV<Object, Object> samzaMessage) {
           .collect(Collectors.toList()));
     } else if (value == null) {
       fieldNames.addAll(relationalSchema.getFieldNames());
-      IntStream.range(0, fieldNames.size() - 1).forEach(x -> values.add(null));
+      IntStream.range(0, fieldNames.size()).forEach(x -> values.add(null));
     } else {
       String msg = "Avro message converter doesn't support messages of type " + value.getClass();
       LOG.error(msg);
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/data/SamzaSqlRelMessage.java b/samza-sql/src/main/java/org/apache/samza/sql/data/SamzaSqlRelMessage.java
index b5df545eaa..bf945a05d4 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/data/SamzaSqlRelMessage.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/data/SamzaSqlRelMessage.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import org.apache.commons.lang.Validate;
 
 
 /**
@@ -51,6 +52,7 @@ public class SamzaSqlRelMessage {
    *               all the fields in the row can be null.
    */
   public SamzaSqlRelMessage(Object key, List<String> names, List<Object> values) {
+    Validate.isTrue(names.size() == values.size(), "Field Names and values are not of same length.");
     this.key = key;
     this.value.addAll(values);
     this.names.addAll(names);
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/fn/FlattenUdf.java b/samza-sql/src/main/java/org/apache/samza/sql/fn/FlattenUdf.java
index 39a26ce751..e8d643dcf2 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/fn/FlattenUdf.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/fn/FlattenUdf.java
@@ -24,7 +24,7 @@
 import org.apache.samza.sql.udfs.ScalarUdf;
 
 
-public class FlattenUdf implements ScalarUdf {
+public class FlattenUdf implements ScalarUdf<Object> {
   @Override
   public void init(Config udfConfig) {
   }
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java b/samza-sql/src/main/java/org/apache/samza/sql/fn/RegexMatchUdf.java
similarity index 88%
rename from samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java
rename to samza-sql/src/main/java/org/apache/samza/sql/fn/RegexMatchUdf.java
index 42e9f04db7..1f5307d958 100644
--- a/samza-tools/src/main/java/org/apache/samza/tools/udf/RegexMatchUdf.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/fn/RegexMatchUdf.java
@@ -17,7 +17,7 @@
 * under the License.
 */
 
-package org.apache.samza.tools.udf;
+package org.apache.samza.sql.fn;
 
 import java.util.regex.Pattern;
 import org.apache.samza.config.Config;
@@ -27,14 +27,13 @@
 /**
  * Simple RegexMatch Udf.
  */
-public class RegexMatchUdf implements ScalarUdf {
+public class RegexMatchUdf implements ScalarUdf<Boolean> {
   @Override
   public void init(Config config) {
 
   }
 
-  @Override
-  public Object execute(Object... args) {
+  public Boolean execute(Object... args) {
     return Pattern.matches((String) args[0], (String) args[1]);
   }
 }
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedUdfResolver.java b/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedUdfResolver.java
index c98a4a1b16..412ff3b8ec 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedUdfResolver.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedUdfResolver.java
@@ -27,7 +27,6 @@
 import java.util.Optional;
 import java.util.Properties;
 import java.util.stream.Collectors;
-
 import org.apache.commons.lang.StringUtils;
 import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
@@ -74,19 +73,20 @@ public ConfigBasedUdfResolver(Properties config, Config udfConfig) {
         throw new SamzaException(msg);
       }
 
-      Optional<Method> udfMethod =
-          Arrays.stream(udfClass.getMethods()).filter(x -> x.getName().equals(UDF_METHOD_NAME)).findFirst();
+      Method udfMethod;
 
-      if (!udfMethod.isPresent()) {
+      try {
+        udfMethod = udfClass.getMethod(UDF_METHOD_NAME, Object[].class);
+      } catch (NoSuchMethodException e) {
         String msg = String.format("Udf Class %s doesn't implement method named %s", udfClassName, UDF_METHOD_NAME);
         LOG.error(msg);
-        throw new SamzaException(msg);
+        throw new SamzaException(msg, e);
       }
 
       int udfIndex = udfClass.getSimpleName().toLowerCase().lastIndexOf("udf");
       String udfName = udfClass.getSimpleName().substring(0, udfIndex);
 
-      udfs.add(new UdfMetadata(udfName, udfMethod.get(), udfConfig.subset(udfName + ".")));
+      udfs.add(new UdfMetadata(udfName, udfMethod, udfConfig.subset(udfName + ".")));
     }
   }
 
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlScalarFunctionImpl.java b/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlScalarFunctionImpl.java
index 1200d57ff0..6894c86ec4 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlScalarFunctionImpl.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlScalarFunctionImpl.java
@@ -67,8 +67,11 @@ public CallImplementor getImplementor() {
       final Expression context = Expressions.parameter(SamzaSqlExecutionContext.class, "context");
       final Expression getUdfInstance = Expressions.call(ScalarUdf.class, context, getUdfMethod,
           Expressions.constant(udfMethod.getDeclaringClass().getName()), Expressions.constant(udfName));
-      return Expressions.call(Expressions.convert_(getUdfInstance, udfMethod.getDeclaringClass()), udfMethod,
-          translatedOperands);
+      final Expression callExpression = Expressions.convert_(Expressions.call(Expressions.convert_(getUdfInstance, udfMethod.getDeclaringClass()), udfMethod,
+          translatedOperands), Object.class);
+      // The Janino compiler which is used to compile the expressions doesn't seem to understand the Type of the ScalarUdf.execute
+      // because it is a generic. To work around that we are explicitly casting it to the return type.
+      return Expressions.convert_(callExpression, udfMethod.getReturnType());
     }, NullPolicy.NONE, false);
   }
 
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/avro/TestAvroRelConversion.java b/samza-sql/src/test/java/org/apache/samza/sql/avro/TestAvroRelConversion.java
index a54db5495c..44b421353d 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/avro/TestAvroRelConversion.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/avro/TestAvroRelConversion.java
@@ -137,6 +137,20 @@ public void testSimpleRecordConversion() {
     LOG.info(Joiner.on(",").join(message.getFieldNames()));
   }
 
+  @Test
+  public void testEmptyRecordConversion() {
+    GenericData.Record record = new GenericData.Record(SimpleRecord.SCHEMA$);
+    SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record));
+    Assert.assertEquals(message.getFieldNames().size(), message.getFieldValues().size());
+  }
+
+
+  @Test
+  public void testNullRecordConversion() {
+    SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", null));
+    Assert.assertEquals(message.getFieldNames().size(), message.getFieldValues().size());
+  }
+
   public static <T> byte[] encodeAvroSpecificRecord(Class<T> clazz, T record) throws IOException {
     DatumWriter<T> msgDatumWriter = new SpecificDatumWriter<>(clazz);
     ByteArrayOutputStream os = new ByteArrayOutputStream();
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/e2e/TestSamzaSqlEndToEnd.java b/samza-sql/src/test/java/org/apache/samza/sql/e2e/TestSamzaSqlEndToEnd.java
index 769c653f16..8baa9e7ea6 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/e2e/TestSamzaSqlEndToEnd.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/e2e/TestSamzaSqlEndToEnd.java
@@ -134,4 +134,20 @@ public void testEndToEndUdf() throws Exception {
     Assert.assertTrue(
         IntStream.range(0, numMessages).map(udf::execute).boxed().collect(Collectors.toList()).equals(outMessages));
   }
+
+  @Test
+  public void testRegexMatchUdfInWhereClause() throws Exception {
+    int numMessages = 20;
+    TestAvroSystemFactory.messages.clear();
+    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(numMessages);
+    String sql1 = "Insert into testavro.outputTopic select id from testavro.SIMPLE1 where RegexMatch('.*4', Name)";
+    List<String> sqlStmts = Collections.singletonList(sql1);
+    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
+    SamzaSqlApplicationRunner runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs));
+    runner.runAndWaitForFinish();
+
+    LOG.info("output Messages " + TestAvroSystemFactory.messages);
+    // There should be two messages that contain "4"
+    Assert.assertEquals(TestAvroSystemFactory.messages.size(), 2);
+  }
 }
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestArrayUdf.java b/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestArrayUdf.java
index c9bc9bcfe8..b2a2c2533f 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestArrayUdf.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestArrayUdf.java
@@ -19,18 +19,19 @@
 
 package org.apache.samza.sql.testutil;
 
+import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import org.apache.samza.config.Config;
 import org.apache.samza.sql.udfs.ScalarUdf;
 
 
-public class MyTestArrayUdf implements ScalarUdf {
+public class MyTestArrayUdf implements ScalarUdf<List<String>> {
   @Override
   public void init(Config udfConfig) {
   }
 
-  public Object execute(Object... args) {
+  public List<String> execute(Object... args) {
     Integer value = (Integer) args[0];
     return IntStream.range(0, value).mapToObj(String::valueOf).collect(Collectors.toList());
   }
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestUdf.java b/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestUdf.java
index baca367260..c718368ba6 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestUdf.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/testutil/MyTestUdf.java
@@ -28,7 +28,7 @@
 /**
  * Test UDF used by unit and integration tests.
  */
-public class MyTestUdf implements ScalarUdf {
+public class MyTestUdf implements ScalarUdf<Integer> {
 
   private static final Logger LOG = LoggerFactory.getLogger(MyTestUdf.class);
 
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java b/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java
index 760d5d0356..008eb82211 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java
@@ -31,6 +31,7 @@
 import org.apache.samza.sql.avro.schemas.ComplexRecord;
 import org.apache.samza.sql.avro.schemas.SimpleRecord;
 import org.apache.samza.sql.fn.FlattenUdf;
+import org.apache.samza.sql.fn.RegexMatchUdf;
 import org.apache.samza.sql.impl.ConfigBasedSourceResolverFactory;
 import org.apache.samza.sql.impl.ConfigBasedUdfResolver;
 import org.apache.samza.sql.interfaces.SqlSystemStreamConfig;
@@ -64,8 +65,9 @@ public static Map<String, String> fetchStaticConfigsWithFactories(int numberOfMe
     String configUdfResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_UDF_RESOLVER_DOMAIN, "config");
     staticConfigs.put(configUdfResolverDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
         ConfigBasedUdfResolver.class.getName());
-    staticConfigs.put(configUdfResolverDomain + ConfigBasedUdfResolver.CFG_UDF_CLASSES,
-        Joiner.on(",").join(MyTestUdf.class.getName(), FlattenUdf.class.getName(), MyTestArrayUdf.class.getName()));
+    staticConfigs.put(configUdfResolverDomain + ConfigBasedUdfResolver.CFG_UDF_CLASSES, Joiner.on(",")
+        .join(MyTestUdf.class.getName(), RegexMatchUdf.class.getName(), FlattenUdf.class.getName(),
+            MyTestArrayUdf.class.getName()));
 
     String avroSystemConfigPrefix =
         String.format(ConfigBasedSourceResolverFactory.CFG_FMT_SAMZA_PREFIX, SAMZA_SYSTEM_TEST_AVRO);
diff --git a/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java b/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java
index 9803117144..11a49f751e 100644
--- a/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java
+++ b/samza-tools/src/main/java/org/apache/samza/tools/SamzaSqlConsole.java
@@ -20,12 +20,6 @@
 package org.apache.samza.tools;
 
 import com.google.common.base.Joiner;
-import org.apache.samza.tools.avro.AvroSchemaGenRelConverterFactory;
-import org.apache.samza.tools.avro.AvroSerDeFactory;
-import org.apache.samza.tools.json.JsonRelConverterFactory;
-import org.apache.samza.tools.schemas.PageViewEvent;
-import org.apache.samza.tools.schemas.ProfileChangeEvent;
-import org.apache.samza.tools.udf.RegexMatchUdf;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -43,6 +37,7 @@
 import org.apache.samza.serializers.StringSerdeFactory;
 import org.apache.samza.sql.avro.ConfigBasedAvroRelSchemaProviderFactory;
 import org.apache.samza.sql.fn.FlattenUdf;
+import org.apache.samza.sql.fn.RegexMatchUdf;
 import org.apache.samza.sql.impl.ConfigBasedSourceResolverFactory;
 import org.apache.samza.sql.impl.ConfigBasedUdfResolver;
 import org.apache.samza.sql.interfaces.SqlSystemStreamConfig;
@@ -52,6 +47,11 @@
 import org.apache.samza.sql.testutil.SqlFileParser;
 import org.apache.samza.standalone.PassthroughJobCoordinatorFactory;
 import org.apache.samza.system.kafka.KafkaSystemFactory;
+import org.apache.samza.tools.avro.AvroSchemaGenRelConverterFactory;
+import org.apache.samza.tools.avro.AvroSerDeFactory;
+import org.apache.samza.tools.json.JsonRelConverterFactory;
+import org.apache.samza.tools.schemas.PageViewEvent;
+import org.apache.samza.tools.schemas.ProfileChangeEvent;
 
 
 public class SamzaSqlConsole {

From 267dfc6ba0eddd78d5913663496c65eaa1aa3c0b Mon Sep 17 00:00:00 2001
From: Srinivasulu Punuru <spunuru@linkedin.com>
Date: Wed, 10 Jan 2018 17:14:03 -0800
Subject: [PATCH 32/36] Fix for the
 TestSamzaSqlApplicationConfig.testConfigInit

Currently testConfigInit checks for a hardcoded number for udfs. Whenever a new UDF is added, This test is going to fail if it is not updated. Changed the test to validate the number of udfs based on the config that is passed.

Author: Srinivasulu Punuru <spunuru@linkedin.com>

Reviewers: Prateek Maheshwari <pmaheshwari@apache.org>

Closes #403 from srinipunuru/testfix.1
---
 .../org/apache/samza/sql/TestSamzaSqlApplicationConfig.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/samza-sql/src/test/java/org/apache/samza/sql/TestSamzaSqlApplicationConfig.java b/samza-sql/src/test/java/org/apache/samza/sql/TestSamzaSqlApplicationConfig.java
index 0b85339928..1c5fc41f62 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/TestSamzaSqlApplicationConfig.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/TestSamzaSqlApplicationConfig.java
@@ -24,6 +24,7 @@
 import junit.framework.Assert;
 import org.apache.samza.SamzaException;
 import org.apache.samza.config.MapConfig;
+import org.apache.samza.sql.impl.ConfigBasedUdfResolver;
 import org.apache.samza.sql.interfaces.SqlSystemStreamConfig;
 import org.apache.samza.sql.runner.SamzaSqlApplicationConfig;
 import org.apache.samza.sql.testutil.SamzaSqlTestConfig;
@@ -36,9 +37,11 @@ public class TestSamzaSqlApplicationConfig {
   public void testConfigInit() {
     Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
     config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.COMPLEX1 select * from testavro.SIMPLE1");
+    String configUdfResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_UDF_RESOLVER_DOMAIN, "config");
+    int numUdfs = config.get(configUdfResolverDomain + ConfigBasedUdfResolver.CFG_UDF_CLASSES).split(",").length;
     SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config));
     Assert.assertEquals(1, samzaSqlApplicationConfig.getQueryInfo().size());
-    Assert.assertEquals(3, samzaSqlApplicationConfig.getUdfMetadata().size());
+    Assert.assertEquals(numUdfs, samzaSqlApplicationConfig.getUdfMetadata().size());
     Assert.assertEquals(1, samzaSqlApplicationConfig.getInputSystemStreamConfigBySource().size());
     Assert.assertEquals(1, samzaSqlApplicationConfig.getOutputSystemStreamConfigsBySource().size());
   }

From 3a9e8064211a79827a2a4793c1f159a689dfa256 Mon Sep 17 00:00:00 2001
From: Prateek Maheshwari <pmaheshw@linkedin.com>
Date: Wed, 17 Jan 2018 09:39:26 -0800
Subject: [PATCH 33/36] SAMZA-1500: Added metrics for RocksDB state store
 memory usage

Approximate RocksDB memory usage = Configured Block Cache size + MemTable size + Indexes and Bloom Filters size =
rocksdb.block-cache-size + rocksdb.size-all-mem-tables + rocksdb.estimate-table-readers-mem

Author: Prateek Maheshwari <pmaheshw@linkedin.com>

Reviewers: Jagadish <jagadish@apache.org>

Closes #404 from prateekm/rocksdb-memory
---
 .../storage/kv/RocksDbOptionsHelper.java      | 12 ++++++---
 .../RocksDbKeyValueStorageEngineFactory.scala |  3 +++
 .../storage/kv/RocksDbKeyValueStore.scala     | 25 +++++++++++++------
 .../storage/kv/TestRocksDbKeyValueStore.scala |  6 ++---
 4 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java
index 9b8f44b195..9389681eca 100644
--- a/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java
+++ b/samza-kv-rocksdb/src/main/java/org/apache/samza/storage/kv/RocksDbOptionsHelper.java
@@ -75,12 +75,10 @@ public static Options options(Config storeConfig, SamzaContainerContext containe
     }
     options.setCompressionType(compressionType);
 
-    Long cacheSize = storeConfig.getLong("container.cache.size.bytes", 100 * 1024 * 1024L);
-    Long cacheSizePerContainer = cacheSize / numTasks;
-
+    long blockCacheSize = getBlockCacheSize(storeConfig, containerContext);
     int blockSize = storeConfig.getInt(ROCKSDB_BLOCK_SIZE_BYTES, 4096);
     BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
-    tableOptions.setBlockCacheSize(cacheSizePerContainer).setBlockSize(blockSize);
+    tableOptions.setBlockCacheSize(blockCacheSize).setBlockSize(blockSize);
     options.setTableFormatConfig(tableOptions);
 
     CompactionStyle compactionStyle = CompactionStyle.UNIVERSAL;
@@ -110,4 +108,10 @@ public static Options options(Config storeConfig, SamzaContainerContext containe
 
     return options;
   }
+
+  public static Long getBlockCacheSize(Config storeConfig, SamzaContainerContext containerContext) {
+    int numTasks = containerContext.taskNames.size();
+    long cacheSize = storeConfig.getLong("container.cache.size.bytes", 100 * 1024 * 1024L);
+    return cacheSize / numTasks;
+  }
 }
\ No newline at end of file
diff --git a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala
index a7b748f173..2b7ffb5977 100644
--- a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala
+++ b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStorageEngineFactory.scala
@@ -44,6 +44,9 @@ class RocksDbKeyValueStorageEngineFactory [K, V] extends BaseKeyValueStorageEngi
     val storageConfig = containerContext.config.subset("stores." + storeName + ".", true)
     val isLoggedStore = containerContext.config.getChangelogStream(storeName).isDefined
     val rocksDbMetrics = new KeyValueStoreMetrics(storeName, registry)
+    rocksDbMetrics.newGauge("rocksdb.block-cache-size",
+      () => RocksDbOptionsHelper.getBlockCacheSize(storageConfig, containerContext))
+
     val rocksDbOptions = RocksDbOptionsHelper.options(storageConfig, containerContext)
     val rocksDbWriteOptions = new WriteOptions().setDisableWAL(true)
     val rocksDbFlushOptions = new FlushOptions().setWaitForFlush(true)
diff --git a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala
index 6aad45f573..eae7da2c1f 100644
--- a/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala
+++ b/samza-kv-rocksdb/src/main/scala/org/apache/samza/storage/kv/RocksDbKeyValueStore.scala
@@ -72,13 +72,24 @@ object RocksDbKeyValueStore extends Logging {
           RocksDB.open(options, dir.toString)
         }
 
-      if (storeConfig.containsKey("rocksdb.metrics.list")) {
-        storeConfig
-          .get("rocksdb.metrics.list")
-          .split(",")
-          .map(property => property.trim)
-          .foreach(property => metrics.newGauge(property, () => rocksDb.getProperty(property)))
-      }
+      // See https://github.com/facebook/rocksdb/blob/master/include/rocksdb/db.h for available properties
+      val rocksDbMetrics = Set (
+        "rocksdb.estimate-table-readers-mem", // indexes and bloom filters
+        "rocksdb.cur-size-active-mem-table", // approximate active memtable size in bytes
+        "rocksdb.cur-size-all-mem-tables", // approximate active and unflushed memtable size in bytes
+        "rocksdb.size-all-mem-tables", // approximate active, unflushed and pinned memtable size in bytes
+        "rocksdb.estimate-num-keys" // approximate number keys in the active and unflushed memtable and storage
+      )
+
+      val configuredMetrics = storeConfig
+        .get("rocksdb.metrics.list", "")
+        .split(",")
+        .map(property => property.trim)
+        .filter(!_.isEmpty)
+        .toSet
+
+      (configuredMetrics ++ rocksDbMetrics)
+        .foreach(property => metrics.newGauge(property, () => rocksDb.getProperty(property)))
 
       rocksDb
     } catch {
diff --git a/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala b/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala
index 418e9867ce..ca9c023ad8 100644
--- a/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala
+++ b/samza-kv-rocksdb/src/test/scala/org/apache/samza/storage/kv/TestRocksDbKeyValueStore.scala
@@ -44,7 +44,7 @@ class TestRocksDbKeyValueStore
                                               config,
                                               false,
                                               "someStore",
-                                              null)
+                                              new KeyValueStoreMetrics())
     val key = "test".getBytes("UTF-8")
     rocksDB.put(key, "val".getBytes("UTF-8"))
     Assert.assertNotNull(rocksDB.get(key))
@@ -76,7 +76,7 @@ class TestRocksDbKeyValueStore
                                               config,
                                               false,
                                               "dbStore",
-                                              null)
+                                              new KeyValueStoreMetrics())
     val key = "key".getBytes("UTF-8")
     rocksDB.put(key, "val".getBytes("UTF-8"))
     // SAMZA-836: Mysteriously,calling new FlushOptions() does not invoke the NativeLibraryLoader in rocksdbjni-3.13.1!
@@ -136,7 +136,7 @@ class TestRocksDbKeyValueStore
                                               config,
                                               false,
                                               "dbStore",
-                                              null)
+                                              new KeyValueStoreMetrics())
 
     val key = "key".getBytes("UTF-8")
     val key1 = "key1".getBytes("UTF-8")

From fcd2b9a53367c2e0b53df7a1fb7eedc8086b285c Mon Sep 17 00:00:00 2001
From: Srinivasulu Punuru <spunuru@linkedin.com>
Date: Wed, 17 Jan 2018 10:06:50 -0800
Subject: [PATCH 34/36] SAMZA-1556: Adding support for multi level sources in
 queries

Right now Samza SQL supports queries with just two levels i.e. `select * from foo.bar`. But there can be sources that are identified though multiple levels. for e.g. `select * from kafka.clusterName.topicName`.

This change adds the support for sql queries with sources that have more than two levels.

Author: Srinivasulu Punuru <spunuru@linkedin.com>

Reviewers: Miguel S<misanchez@linkedin.com>, Aditya T<atoomula@linkedin.com>

Closes #405 from srinipunuru/multi-level.1
---
 .../ConfigBasedSourceResolverFactory.java     |  8 +--
 .../sql/interfaces/SqlSystemStreamConfig.java | 47 ++++++++++++++++-
 .../samza/sql/planner/QueryPlanner.java       | 38 +++++++++-----
 .../sql/runner/SamzaSqlApplicationConfig.java | 23 ++++-----
 .../sql/runner/SamzaSqlApplicationRunner.java |  3 ++
 .../samza/sql/translator/QueryTranslator.java |  5 +-
 .../samza/sql/translator/ScanTranslator.java  | 26 ++++------
 .../apache/samza/sql/TestQueryTranslator.java |  2 +-
 .../sql/testutil/SamzaSqlTestConfig.java      |  2 +-
 .../testutil/TestSourceResolverFactory.java   | 50 +++++++++++++++++++
 10 files changed, 154 insertions(+), 50 deletions(-)
 create mode 100644 samza-sql/src/test/java/org/apache/samza/sql/testutil/TestSourceResolverFactory.java

diff --git a/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedSourceResolverFactory.java b/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedSourceResolverFactory.java
index 1203b254ca..a2d8b0c04f 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedSourceResolverFactory.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/impl/ConfigBasedSourceResolverFactory.java
@@ -29,8 +29,8 @@
 
 
 /**
- * Schema Resolver that uses static config to return a config corresponding to a system stream.
- * Schemas are configured using the config of format {systemName}.{streamName}.schema.
+ * Source Resolver implementation that uses static config to return a config corresponding to a system stream.
+ * This Source resolver implementation supports sources of type {systemName}.{streamName}
  */
 public class ConfigBasedSourceResolverFactory implements SourceResolverFactory {
 
@@ -53,8 +53,10 @@ public ConfigBasedSourceResolver(Config config) {
     @Override
     public SqlSystemStreamConfig fetchSourceInfo(String source) {
       String[] sourceComponents = source.split("\\.");
+
+      // This source resolver expects sources of format {systemName}.{streamName}
       if (sourceComponents.length != 2) {
-        String msg = String.format("Source %s is not of the format <system>.<stream>", source);
+        String msg = String.format("Source %s is not of the format {systemName}.{streamName{", source);
         LOG.error(msg);
         throw new SamzaException(msg);
       }
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlSystemStreamConfig.java b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlSystemStreamConfig.java
index df21784447..d8965a43f6 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlSystemStreamConfig.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlSystemStreamConfig.java
@@ -19,8 +19,13 @@
 
 package org.apache.samza.sql.interfaces;
 
+import com.google.common.base.Joiner;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import org.apache.commons.lang.Validate;
 import org.apache.samza.config.Config;
+import org.apache.samza.config.MapConfig;
 import org.apache.samza.system.SystemStream;
 
 
@@ -38,18 +43,48 @@ public class SqlSystemStreamConfig {
 
   private final String samzaRelConverterName;
   private final SystemStream systemStream;
+
+  private final String source;
   private String relSchemaProviderName;
 
+  private Config config;
+
+  private List<String> sourceParts;
+
   public SqlSystemStreamConfig(String systemName, String streamName, Config systemConfig) {
+    this(systemName, streamName, Arrays.asList(systemName, streamName), systemConfig);
+  }
+
+  public SqlSystemStreamConfig(String systemName, String streamName, List<String> sourceParts,
+      Config systemConfig) {
 
+
+    HashMap<String, String> streamConfigs = new HashMap<>(systemConfig);
     this.systemName = systemName;
     this.streamName = streamName;
+    this.source = getSourceFromSourceParts(sourceParts);
+    this.sourceParts = sourceParts;
     this.systemStream = new SystemStream(systemName, streamName);
 
-    samzaRelConverterName = systemConfig.get(CFG_SAMZA_REL_CONVERTER);
-    relSchemaProviderName = systemConfig.get(CFG_REL_SCHEMA_PROVIDER);
+    samzaRelConverterName = streamConfigs.get(CFG_SAMZA_REL_CONVERTER);
     Validate.notEmpty(samzaRelConverterName,
         String.format("%s is not set or empty for system %s", CFG_SAMZA_REL_CONVERTER, systemName));
+
+    relSchemaProviderName = streamConfigs.get(CFG_REL_SCHEMA_PROVIDER);
+
+    // Removing the Samza SQL specific configs to get the remaining Samza configs.
+    streamConfigs.remove(CFG_SAMZA_REL_CONVERTER);
+    streamConfigs.remove(CFG_REL_SCHEMA_PROVIDER);
+
+    config = new MapConfig(streamConfigs);
+  }
+
+  public static String getSourceFromSourceParts(List<String> sourceParts) {
+    return Joiner.on(".").join(sourceParts);
+  }
+
+  public List<String> getSourceParts() {
+    return sourceParts;
   }
 
   public String getSystemName() {
@@ -71,4 +106,12 @@ public String getRelSchemaProviderName() {
   public SystemStream getSystemStream() {
     return systemStream;
   }
+
+  public Config getConfig() {
+    return config;
+  }
+
+  public String getSource() {
+    return source;
+  }
 }
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/planner/QueryPlanner.java b/samza-sql/src/main/java/org/apache/samza/sql/planner/QueryPlanner.java
index ce03ba3166..061c03fd5a 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/planner/QueryPlanner.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/planner/QueryPlanner.java
@@ -23,7 +23,6 @@
 import java.sql.DriverManager;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -69,11 +68,16 @@
 public class QueryPlanner {
   private static final Logger LOG = LoggerFactory.getLogger(QueryPlanner.class);
 
-  private final Map<String, SqlSystemStreamConfig> systemStreamConfigBySource;
   private final Collection<UdfMetadata> udfMetadata;
-  private final Map<SystemStream, RelSchemaProvider> relSchemaProviders;
 
-  public QueryPlanner(Map<SystemStream, RelSchemaProvider> relSchemaProviders,
+  // Mapping between the source to the RelSchemaProvider corresponding to the source.
+  private final Map<String, RelSchemaProvider> relSchemaProviders;
+
+  // Mapping between the source to the SqlSystemStreamConfig corresponding to the source.
+  private final Map<String, SqlSystemStreamConfig> systemStreamConfigBySource;
+
+
+  public QueryPlanner(Map<String, RelSchemaProvider> relSchemaProviders,
       Map<String, SqlSystemStreamConfig> systemStreamConfigBySource, Collection<UdfMetadata> udfMetadata) {
     this.relSchemaProviders = relSchemaProviders;
     this.systemStreamConfigBySource = systemStreamConfigBySource;
@@ -85,16 +89,26 @@ public RelRoot plan(String query) {
       Connection connection = DriverManager.getConnection("jdbc:calcite:");
       CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
       SchemaPlus rootSchema = calciteConnection.getRootSchema();
-      Map<String, SchemaPlus> systemSchemas = new HashMap<>();
 
       for (SqlSystemStreamConfig ssc : systemStreamConfigBySource.values()) {
-
-        RelSchemaProvider relSchemaProvider = relSchemaProviders.get(ssc.getSystemStream());
-        SchemaPlus systemSchema =
-            systemSchemas.computeIfAbsent(ssc.getSystemName(), s -> rootSchema.add(s, new AbstractSchema()));
-        RelDataType relationalSchema = relSchemaProvider.getRelationalSchema();
-
-        systemSchema.add(ssc.getStreamName(), createTableFromRelSchema(relationalSchema));
+        SchemaPlus previousLevelSchema = rootSchema;
+        List<String> sourceParts = ssc.getSourceParts();
+        RelSchemaProvider relSchemaProvider = relSchemaProviders.get(ssc.getSource());
+
+        for (String sourcePart : sourceParts) {
+          if (!sourcePart.equalsIgnoreCase(ssc.getStreamName())) {
+            SchemaPlus sourcePartSchema = rootSchema.getSubSchema(sourcePart);
+            if (sourcePartSchema == null) {
+              sourcePartSchema = previousLevelSchema.add(sourcePart, new AbstractSchema());
+            }
+            previousLevelSchema = sourcePartSchema;
+          } else {
+            // If the source part is the streamName, then fetch the schema corresponding to the stream and register.
+            RelDataType relationalSchema = relSchemaProvider.getRelationalSchema();
+            previousLevelSchema.add(ssc.getStreamName(), createTableFromRelSchema(relationalSchema));
+            break;
+          }
+        }
       }
 
       List<SamzaSqlScalarFunctionImpl> samzaSqlFunctions = udfMetadata.stream()
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationConfig.java b/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationConfig.java
index f0297452a6..227a0f1a85 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationConfig.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationConfig.java
@@ -78,8 +78,8 @@ public class SamzaSqlApplicationConfig {
 
   public static final String CFG_UDF_RESOLVER = "samza.sql.udfResolver";
   public static final String CFG_FMT_UDF_RESOLVER_DOMAIN = "samza.sql.udfResolver.%s.";
-  private final Map<SystemStream, RelSchemaProvider> relSchemaProvidersBySystemStream;
-  private final Map<SystemStream, SamzaRelConverter> samzaRelConvertersBySystemStream;
+  private final Map<String, RelSchemaProvider> relSchemaProvidersBySource;
+  private final Map<String, SamzaRelConverter> samzaRelConvertersBySource;
 
   private SourceResolver sourceResolver;
   private UdfResolver udfResolver;
@@ -87,7 +87,6 @@ public class SamzaSqlApplicationConfig {
   private final Collection<UdfMetadata> udfMetadata;
 
   private final Map<String, SqlSystemStreamConfig> inputSystemStreamConfigBySource;
-
   private final Map<String, SqlSystemStreamConfig> outputSystemStreamConfigsBySource;
 
   private final List<String> sql;
@@ -117,17 +116,17 @@ public SamzaSqlApplicationConfig(Config staticConfig) {
         .collect(Collectors.toMap(Function.identity(), x -> sourceResolver.fetchSourceInfo(x)));
     systemStreamConfigs.addAll(outputSystemStreamConfigsBySource.values());
 
-    relSchemaProvidersBySystemStream = systemStreamConfigs.stream()
-        .collect(Collectors.toMap(SqlSystemStreamConfig::getSystemStream,
+    relSchemaProvidersBySource = systemStreamConfigs.stream()
+        .collect(Collectors.toMap(SqlSystemStreamConfig::getSource,
             x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig,
                 CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN,
                 (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c))));
 
-    samzaRelConvertersBySystemStream = systemStreamConfigs.stream()
-        .collect(Collectors.toMap(SqlSystemStreamConfig::getSystemStream,
+    samzaRelConvertersBySource = systemStreamConfigs.stream()
+        .collect(Collectors.toMap(SqlSystemStreamConfig::getSource,
             x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig,
                 CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(),
-                    relSchemaProvidersBySystemStream.get(x.getSystemStream()), c))));
+                    relSchemaProvidersBySource.get(x.getSource()), c))));
   }
 
   private static <T> T initializePlugin(String pluginName, String plugin, Config staticConfig,
@@ -235,11 +234,11 @@ public Map<String, SqlSystemStreamConfig> getOutputSystemStreamConfigsBySource()
     return outputSystemStreamConfigsBySource;
   }
 
-  public Map<SystemStream, SamzaRelConverter> getSamzaRelConverters() {
-    return samzaRelConvertersBySystemStream;
+  public Map<String, SamzaRelConverter> getSamzaRelConverters() {
+    return samzaRelConvertersBySource;
   }
 
-  public Map<SystemStream, RelSchemaProvider> getRelSchemaProviders() {
-    return relSchemaProvidersBySystemStream;
+  public Map<String, RelSchemaProvider> getRelSchemaProviders() {
+    return relSchemaProvidersBySource;
   }
 }
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationRunner.java b/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationRunner.java
index 57d889f1f3..5215f7e47a 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationRunner.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/runner/SamzaSqlApplicationRunner.java
@@ -85,10 +85,13 @@ public static Config computeSamzaConfigs(Boolean localRunner, Config config) {
         SqlSystemStreamConfig inputSystemStreamConfig = sourceResolver.fetchSourceInfo(inputSource);
         newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, inputSystemStreamConfig.getStreamName()),
             inputSystemStreamConfig.getSystemName());
+        newConfig.putAll(inputSystemStreamConfig.getConfig());
       }
+
       SqlSystemStreamConfig outputSystemStreamConfig = sourceResolver.fetchSourceInfo(query.getOutputSource());
       newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, outputSystemStreamConfig.getStreamName()),
           outputSystemStreamConfig.getSystemName());
+      newConfig.putAll(outputSystemStreamConfig.getConfig());
     }
 
     if (localRunner) {
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/translator/QueryTranslator.java b/samza-sql/src/main/java/org/apache/samza/sql/translator/QueryTranslator.java
index ab17018075..87e37f4f19 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/translator/QueryTranslator.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/translator/QueryTranslator.java
@@ -50,7 +50,8 @@ public class QueryTranslator {
 
   public QueryTranslator(SamzaSqlApplicationConfig sqlConfig) {
     this.sqlConfig = sqlConfig;
-    scanTranslator = new ScanTranslator(sqlConfig.getSamzaRelConverters());
+    scanTranslator =
+        new ScanTranslator(sqlConfig.getSamzaRelConverters(), sqlConfig.getInputSystemStreamConfigBySource());
   }
 
   public void translate(SamzaSqlQueryParser.QueryInfo queryInfo, StreamGraph streamGraph) {
@@ -87,7 +88,7 @@ public RelNode visit(LogicalProject project) {
 
     SqlSystemStreamConfig outputSystemConfig =
         sqlConfig.getOutputSystemStreamConfigsBySource().get(queryInfo.getOutputSource());
-    SamzaRelConverter samzaMsgConverter = sqlConfig.getSamzaRelConverters().get(outputSystemConfig.getSystemStream());
+    SamzaRelConverter samzaMsgConverter = sqlConfig.getSamzaRelConverters().get(queryInfo.getOutputSource());
     MessageStreamImpl<SamzaSqlRelMessage> stream =
         (MessageStreamImpl<SamzaSqlRelMessage>) context.getMessageStream(node.getId());
     MessageStream<KV<Object, Object>> outputStream = stream.map(samzaMsgConverter::convertToSamzaMessage);
diff --git a/samza-sql/src/main/java/org/apache/samza/sql/translator/ScanTranslator.java b/samza-sql/src/main/java/org/apache/samza/sql/translator/ScanTranslator.java
index 202bdbd1ab..30e5a9b3d2 100644
--- a/samza-sql/src/main/java/org/apache/samza/sql/translator/ScanTranslator.java
+++ b/samza-sql/src/main/java/org/apache/samza/sql/translator/ScanTranslator.java
@@ -21,19 +21,14 @@
 
 import java.util.List;
 import java.util.Map;
-
 import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.type.RelDataType;
 import org.apache.commons.lang.Validate;
 import org.apache.samza.operators.KV;
 import org.apache.samza.operators.MessageStream;
 import org.apache.samza.operators.StreamGraph;
-import org.apache.samza.operators.functions.MapFunction;
 import org.apache.samza.sql.data.SamzaSqlRelMessage;
 import org.apache.samza.sql.interfaces.SamzaRelConverter;
-
-import com.google.common.base.Joiner;
-import org.apache.samza.system.SystemStream;
+import org.apache.samza.sql.interfaces.SqlSystemStreamConfig;
 
 
 /**
@@ -42,25 +37,22 @@
  */
 public class ScanTranslator {
 
-  private final Map<SystemStream, SamzaRelConverter> relMsgConverters;
+  private final Map<String, SamzaRelConverter> relMsgConverters;
+  private final Map<String, SqlSystemStreamConfig> systemStreamConfig;
 
-  public ScanTranslator(Map<SystemStream, SamzaRelConverter> converters) {
+  public ScanTranslator(Map<String, SamzaRelConverter> converters, Map<String, SqlSystemStreamConfig> ssc) {
     relMsgConverters = converters;
+    this.systemStreamConfig = ssc;
   }
 
   public void translate(final TableScan tableScan, final TranslatorContext context) {
     StreamGraph streamGraph = context.getStreamGraph();
     List<String> tableNameParts = tableScan.getTable().getQualifiedName();
-    Validate.isTrue(tableNameParts.size() == 2,
-        String.format("table name %s is not of the format <SystemName>.<StreamName>",
-            Joiner.on(".").join(tableNameParts)));
-
-    String streamName = tableNameParts.get(1);
-    String systemName = tableNameParts.get(0);
-    SystemStream systemStream = new SystemStream(systemName, streamName);
+    String sourceName = SqlSystemStreamConfig.getSourceFromSourceParts(tableNameParts);
 
-    Validate.isTrue(relMsgConverters.containsKey(systemStream), String.format("Unknown system %s", systemName));
-    SamzaRelConverter converter = relMsgConverters.get(systemStream);
+    Validate.isTrue(relMsgConverters.containsKey(sourceName), String.format("Unknown source %s", sourceName));
+    SamzaRelConverter converter = relMsgConverters.get(sourceName);
+    String streamName = systemStreamConfig.get(sourceName).getStreamName();
 
     MessageStream<KV<Object, Object>> inputStream = streamGraph.getInputStream(streamName);
     MessageStream<SamzaSqlRelMessage> samzaSqlRelMessageStream = inputStream.map(converter::convertToRelMessage);
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/TestQueryTranslator.java b/samza-sql/src/test/java/org/apache/samza/sql/TestQueryTranslator.java
index 7fa99749c5..04fdec55a7 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/TestQueryTranslator.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/TestQueryTranslator.java
@@ -39,7 +39,7 @@ public class TestQueryTranslator {
   public void testTranslate() {
     Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
     config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT,
-        "Insert into testavro.outputTopic select MyTest(id) from testavro.SIMPLE1");
+        "Insert into testavro.outputTopic select MyTest(id) from testavro.level1.level2.SIMPLE1");
     Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
     SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config));
     QueryTranslator translator = new QueryTranslator(samzaSqlApplicationConfig);
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java b/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java
index 008eb82211..92766f66d2 100644
--- a/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java
+++ b/samza-sql/src/test/java/org/apache/samza/sql/testutil/SamzaSqlTestConfig.java
@@ -59,7 +59,7 @@ public static Map<String, String> fetchStaticConfigsWithFactories(int numberOfMe
     String configSourceResolverDomain =
         String.format(SamzaSqlApplicationConfig.CFG_FMT_SOURCE_RESOLVER_DOMAIN, "config");
     staticConfigs.put(configSourceResolverDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
-        ConfigBasedSourceResolverFactory.class.getName());
+        TestSourceResolverFactory.class.getName());
 
     staticConfigs.put(SamzaSqlApplicationConfig.CFG_UDF_RESOLVER, "config");
     String configUdfResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_UDF_RESOLVER_DOMAIN, "config");
diff --git a/samza-sql/src/test/java/org/apache/samza/sql/testutil/TestSourceResolverFactory.java b/samza-sql/src/test/java/org/apache/samza/sql/testutil/TestSourceResolverFactory.java
new file mode 100644
index 0000000000..b9cf8032e2
--- /dev/null
+++ b/samza-sql/src/test/java/org/apache/samza/sql/testutil/TestSourceResolverFactory.java
@@ -0,0 +1,50 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+package org.apache.samza.sql.testutil;
+
+import java.util.Arrays;
+import org.apache.samza.config.Config;
+import org.apache.samza.sql.interfaces.SourceResolver;
+import org.apache.samza.sql.interfaces.SourceResolverFactory;
+import org.apache.samza.sql.interfaces.SqlSystemStreamConfig;
+
+
+public class TestSourceResolverFactory implements SourceResolverFactory {
+  @Override
+  public SourceResolver create(Config config) {
+    return new TestSourceResolver(config);
+  }
+
+  private class TestSourceResolver implements SourceResolver {
+    private final Config config;
+
+    public TestSourceResolver(Config config) {
+      this.config = config;
+    }
+
+    @Override
+    public SqlSystemStreamConfig fetchSourceInfo(String sourceName) {
+      String[] sourceComponents = sourceName.split("\\.");
+      Config systemConfigs = config.subset(sourceComponents[0] + ".");
+      return new SqlSystemStreamConfig(sourceComponents[0], sourceComponents[sourceComponents.length - 1],
+          Arrays.asList(sourceComponents), systemConfigs);
+    }
+  }
+}

From 95a71fb9e234c9fc7fcb8a4ec0bbab33c9555738 Mon Sep 17 00:00:00 2001
From: navina <navina@apache.org>
Date: Wed, 17 Jan 2018 10:11:40 -0800
Subject: [PATCH 35/36] SAMZA-1523: Cleanup table entries before shutting down
 the processor

Modified the `TableUtils#deleteProcessorEntity` to provide an option to disable optimistic locking during a call to Azure Table Storage service.

sborya PawasChhokra nickpan47   Review please?

Author: navina <navina@apache.org>

Reviewers: Shanthoosh V<svenkata@linkedin.com>, Boris S<bshkolni@linkedin.com>

Closes #379 from navina/azure-etag-fix
---
 .../coordinator/AzureJobCoordinator.java      | 75 ++++++++++---------
 .../coordinator/data/ProcessorEntity.java     |  8 ++
 .../scheduler/JMVersionUpgradeScheduler.java  |  9 ++-
 .../org/apache/samza/util/TableUtils.java     | 49 +++++++++---
 4 files changed, 89 insertions(+), 52 deletions(-)

diff --git a/samza-azure/src/main/java/org/apache/samza/coordinator/AzureJobCoordinator.java b/samza-azure/src/main/java/org/apache/samza/coordinator/AzureJobCoordinator.java
index 468705b9ef..ca3384dc9a 100644
--- a/samza-azure/src/main/java/org/apache/samza/coordinator/AzureJobCoordinator.java
+++ b/samza-azure/src/main/java/org/apache/samza/coordinator/AzureJobCoordinator.java
@@ -19,22 +19,10 @@
 
 package org.apache.samza.coordinator;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.samza.AzureClient;
-import org.apache.samza.config.AzureConfig;
-import org.apache.samza.coordinator.data.BarrierState;
 import org.apache.samza.config.ApplicationConfig;
+import org.apache.samza.config.AzureConfig;
 import org.apache.samza.config.Config;
 import org.apache.samza.config.ConfigException;
 import org.apache.samza.config.JobConfig;
@@ -42,8 +30,8 @@
 import org.apache.samza.container.TaskName;
 import org.apache.samza.container.grouper.stream.SystemStreamPartitionGrouper;
 import org.apache.samza.container.grouper.stream.SystemStreamPartitionGrouperFactory;
-import org.apache.samza.job.model.JobModel;
-import org.apache.samza.runtime.ProcessorIdGenerator;
+import org.apache.samza.coordinator.data.BarrierState;
+import org.apache.samza.coordinator.data.ProcessorEntity;
 import org.apache.samza.coordinator.scheduler.HeartbeatScheduler;
 import org.apache.samza.coordinator.scheduler.JMVersionUpgradeScheduler;
 import org.apache.samza.coordinator.scheduler.LeaderBarrierCompleteScheduler;
@@ -51,6 +39,8 @@
 import org.apache.samza.coordinator.scheduler.LivenessCheckScheduler;
 import org.apache.samza.coordinator.scheduler.RenewLeaseScheduler;
 import org.apache.samza.coordinator.scheduler.SchedulerStateChangeListener;
+import org.apache.samza.job.model.JobModel;
+import org.apache.samza.runtime.ProcessorIdGenerator;
 import org.apache.samza.system.StreamMetadataCache;
 import org.apache.samza.system.SystemStream;
 import org.apache.samza.system.SystemStreamMetadata;
@@ -64,6 +54,18 @@
 import org.slf4j.LoggerFactory;
 import scala.collection.JavaConverters;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 
 /**
  * Class that provides coordination mechanism for Samza standalone in Azure.
@@ -80,7 +82,6 @@ public class AzureJobCoordinator implements JobCoordinator {
   private final TableUtils table;
   private final Config config;
   private final String processorId;
-  private final AzureClient client;
   private final AtomicReference<String> currentJMVersion;
   private final AtomicBoolean versionUpgradeDetected;
   private final HeartbeatScheduler heartbeat;
@@ -103,7 +104,7 @@ public AzureJobCoordinator(Config config) {
     processorId = createProcessorId(config);
     currentJMVersion = new AtomicReference<>(INITIAL_STATE);
     AzureConfig azureConfig = new AzureConfig(config);
-    client = new AzureClient(azureConfig.getAzureConnectionString());
+    AzureClient client = new AzureClient(azureConfig.getAzureConnectionString());
     leaderBlob = new BlobUtils(client, azureConfig.getAzureContainerName(), azureConfig.getAzureBlobName(), azureConfig.getAzureBlobLength());
     errorHandler = (errorMsg) -> {
       LOG.error(errorMsg);
@@ -149,18 +150,17 @@ public void start() {
   public void stop() {
     LOG.info("Shutting down Azure job coordinator.");
 
-    if (coordinatorListener != null) {
-      coordinatorListener.onJobModelExpired();
-    }
-
-    // Resign leadership
-    if (azureLeaderElector.amILeader()) {
-      azureLeaderElector.resignLeadership();
-    }
+    // Clean up resources & Resign leadership (if you are leader)
+    azureLeaderElector.resignLeadership();
+    table.deleteProcessorEntity(currentJMVersion.get(), processorId, true);
 
     // Shutdown all schedulers
     shutdownSchedulers();
 
+    if (coordinatorListener != null) {
+      coordinatorListener.onJobModelExpired();
+    }
+
     if (coordinatorListener != null) {
       coordinatorListener.onCoordinatorStop();
     }
@@ -217,7 +217,6 @@ private SchedulerStateChangeListener createLeaderBarrierCompleteListener(String
       if (!leaderBlob.publishBarrierState(state, azureLeaderElector.getLeaseId().get())) {
         LOG.info("Leader failed to publish the job model {}. Stopping the processor with PID: .", jobModel, processorId);
         stop();
-        table.deleteProcessorEntity(currentJMVersion.get(), processorId);
       }
       leaderBarrierScheduler.shutdown();
     };
@@ -374,7 +373,6 @@ private void doOnProcessorChange(List<String> currentProcessorIds) {
     if (!jmWrite || !barrierWrite || !processorWrite) {
       LOG.info("Leader failed to publish the job model {}. Stopping the processor with PID: .", jobModel, processorId);
       stop();
-      table.deleteProcessorEntity(currentJMVersion.get(), processorId);
     }
 
     LOG.info("pid=" + processorId + "Published new Job Model. Version = " + nextJMVersion);
@@ -400,7 +398,6 @@ private void onNewJobModelAvailable(final String nextJMVersion) {
     if (!jobModel.getContainers().containsKey(processorId)) {
       LOG.info("JobModel: {} does not contain the processorId: {}. Stopping the processor.", jobModel, processorId);
       stop();
-      table.deleteProcessorEntity(currentJMVersion.get(), processorId);
     } else {
       //Stop current work
       if (coordinatorListener != null) {
@@ -443,18 +440,24 @@ private void onNewJobModelAvailable(final String nextJMVersion) {
   private void onNewJobModelConfirmed(final String nextJMVersion) {
     LOG.info("pid=" + processorId + "new version " + nextJMVersion + " of the job model got confirmed");
 
-    // Delete previous value
-    if (table.getEntity(currentJMVersion.get(), processorId) != null) {
-      table.deleteProcessorEntity(currentJMVersion.get(), processorId);
-    }
-    if (table.getEntity(INITIAL_STATE, processorId) != null) {
-      table.deleteProcessorEntity(INITIAL_STATE, processorId);
-    }
+    String prevVersion = currentJMVersion.get();
 
-    //Start heartbeating to new entry only when barrier reached.
+    //Start heart-beating to new entry only when barrier reached.
     //Changing the current job model version enables that since we are heartbeating to a row identified by the current job model version.
     currentJMVersion.getAndSet(nextJMVersion);
 
+    // Delete previous value
+    ProcessorEntity entity = table.getEntity(prevVersion, processorId);
+    if (entity != null) {
+      entity.setEtag("*");
+      table.deleteProcessorEntity(entity);
+    }
+    entity = table.getEntity(INITIAL_STATE, processorId);
+    if (entity != null) {
+      entity.setEtag("*");
+      table.deleteProcessorEntity(entity);
+    }
+
     //Start the container with the new model
     if (coordinatorListener != null) {
       coordinatorListener.onNewJobModel(processorId, jobModel);
diff --git a/samza-azure/src/main/java/org/apache/samza/coordinator/data/ProcessorEntity.java b/samza-azure/src/main/java/org/apache/samza/coordinator/data/ProcessorEntity.java
index 9323bde35e..27c18b35da 100644
--- a/samza-azure/src/main/java/org/apache/samza/coordinator/data/ProcessorEntity.java
+++ b/samza-azure/src/main/java/org/apache/samza/coordinator/data/ProcessorEntity.java
@@ -59,4 +59,12 @@ public void setIsLeader(boolean leader) {
   public boolean getIsLeader() {
     return isLeader;
   }
+
+  public String getJobModelVersion() {
+    return partitionKey;
+  }
+
+  public String getProcessorId() {
+    return rowKey;
+  }
 }
\ No newline at end of file
diff --git a/samza-azure/src/main/java/org/apache/samza/coordinator/scheduler/JMVersionUpgradeScheduler.java b/samza-azure/src/main/java/org/apache/samza/coordinator/scheduler/JMVersionUpgradeScheduler.java
index ded014ff64..235b1f8d95 100644
--- a/samza-azure/src/main/java/org/apache/samza/coordinator/scheduler/JMVersionUpgradeScheduler.java
+++ b/samza-azure/src/main/java/org/apache/samza/coordinator/scheduler/JMVersionUpgradeScheduler.java
@@ -20,6 +20,11 @@
 package org.apache.samza.coordinator.scheduler;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.samza.coordinator.data.BarrierState;
+import org.apache.samza.util.BlobUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -29,10 +34,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
-import org.apache.samza.coordinator.data.BarrierState;
-import org.apache.samza.util.BlobUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 /**
diff --git a/samza-azure/src/main/java/org/apache/samza/util/TableUtils.java b/samza-azure/src/main/java/org/apache/samza/util/TableUtils.java
index f49ce27b47..d5b8320168 100644
--- a/samza-azure/src/main/java/org/apache/samza/util/TableUtils.java
+++ b/samza-azure/src/main/java/org/apache/samza/util/TableUtils.java
@@ -24,17 +24,17 @@
 import com.microsoft.azure.storage.table.CloudTableClient;
 import com.microsoft.azure.storage.table.TableOperation;
 import com.microsoft.azure.storage.table.TableQuery;
-import java.net.URISyntaxException;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
 import org.apache.samza.AzureClient;
 import org.apache.samza.AzureException;
 import org.apache.samza.coordinator.data.ProcessorEntity;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
 
 /**
  *  Client side class that has a reference to Azure Table Storage.
@@ -44,17 +44,15 @@
  *  ROW KEY = Unique entity ID for a group = Processor ID (for this case).
  */
 public class TableUtils {
-
   private static final Logger LOG = LoggerFactory.getLogger(TableUtils.class);
   private static final String PARTITION_KEY = "PartitionKey";
   private static final long LIVENESS_DEBOUNCE_TIME_SEC = 30;
   private final String initialState;
-  private final CloudTableClient tableClient;
   private final CloudTable table;
 
   public TableUtils(AzureClient client, String tableName, String initialState) {
     this.initialState = initialState;
-    tableClient = client.getTableClient();
+    CloudTableClient tableClient = client.getTableClient();
     try {
       table = tableClient.getTableReference(tableName);
       table.createIfNotExists();
@@ -97,8 +95,7 @@ public void addProcessorEntity(String jmVersion, String pid, boolean isLeader) {
   public ProcessorEntity getEntity(String jmVersion, String pid) {
     try {
       TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
-      ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType();
-      return entity;
+      return table.execute(retrieveEntity).getResultAsType();
     } catch (StorageException e) {
       LOG.error("Azure storage exception while retrieving processor entity with job model version: " + jmVersion + "and pid: " + pid, e);
       throw new AzureException(e);
@@ -112,7 +109,6 @@ public ProcessorEntity getEntity(String jmVersion, String pid) {
    */
   public void updateHeartbeat(String jmVersion, String pid) {
     try {
-      Random rand = new Random();
       TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
       ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType();
       entity.updateLiveness();
@@ -145,14 +141,24 @@ public void updateIsLeader(String jmVersion, String pid, boolean isLeader) {
 
   /**
    * Deletes a specified row in the processor table.
+   *
+   * Note: Table service uses optimistic locking by default. Hence, if there is an update after retrieving the entity,
+   * then the delete operation will fail.
+   *
    * @param jmVersion Job model version of the processor row to be deleted.
    * @param pid Unique processor ID of the processor row to be deleted.
+   * @param force True, to disable optimistic locking on the table. False, otherwise. Setting to false may result in
+   *              AzureException when there is concurrent access to the table.
+   *
    * @throws AzureException If an Azure storage service error occurred.
    */
-  public void deleteProcessorEntity(String jmVersion, String pid) {
+  public void deleteProcessorEntity(String jmVersion, String pid, boolean force) {
     try {
       TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class);
       ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType();
+      if (force) {
+        entity.setEtag("*");
+      }
       TableOperation remove = TableOperation.delete(entity);
       table.execute(remove);
     } catch (StorageException e) {
@@ -161,6 +167,25 @@ public void deleteProcessorEntity(String jmVersion, String pid) {
     }
   }
 
+  /**
+   * Deletes a specified row in the processor table.
+   *
+   * Note: Table service uses optimistic locking by default. In order to disable it, set the ETag on the ProcessorEntity
+   * to "*" before invoking this method.
+   *
+   * @param entity ProcessorEntity that has to be deleted
+   * @throws AzureException If an Azure storage service error occurred.
+   */
+  public void deleteProcessorEntity(ProcessorEntity entity) {
+    try {
+      TableOperation remove = TableOperation.delete(entity);
+      table.execute(remove);
+    } catch (StorageException e) {
+      LOG.error("Azure storage exception while deleting processor entity with job model version: " +
+          entity.getJobModelVersion() + "and pid: " + entity.getProcessorId(), e);
+      throw new AzureException(e);
+    }
+  }
   /**
    * Retrieve all rows in a table with the given partition key.
    * @param partitionKey Job model version of the processors to be retrieved.

From dd42bc3dfe9ab203d19752daf124906eaabd4559 Mon Sep 17 00:00:00 2001
From: Fred Ji <haifeng.ji@gmail.com>
Date: Thu, 30 Nov 2017 15:29:25 -0800
Subject: [PATCH 36/36] SAMZA-1407 upgrade junit version to 4.12

---
 gradle/dependency-versions.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle
index 2e45914f59..6ee117f48c 100644
--- a/gradle/dependency-versions.gradle
+++ b/gradle/dependency-versions.gradle
@@ -32,7 +32,7 @@
   jerseyVersion = "2.22.1"
   jodaTimeVersion = "2.2"
   joptSimpleVersion = "3.2"
-  junitVersion = "4.8.1"
+  junitVersion = "4.12"
   kafkaVersion = "0.11.0.2"
   log4jVersion = "1.2.17"
   metricsVersion = "2.2.0"