Add Makefile, input data and expected test outputs (#99)

.semaphore/semaphore.yml

@@ -139,7 +139,7 @@ blocks:
         - name: KStreams finding distinct events tests
           commands:
             - make -C _includes/tutorials/finding-distinct/kstreams/code tutorial
-#
-#        - name: KStreams emit a final result from a time window
-#          commands:
-#            - make -C _includes/tutorials/window-final-result/kstreams/code tutorial
+
+        - name: KStreams emit a final result from a time window
+          commands:
+            - make -C _includes/tutorials/window-final-result/kstreams/code tutorial

_data/harnesses/window-final-result/kstreams.yml

@@ -13,6 +13,17 @@ dev:
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/1_1-make-docker-compose.adoc
 
+        - action: execute_async
+          file: tutorial-steps/dev/docker-compose-up.sh
+          render:
+            file: tutorials/window-final-result/kstreams/markup/dev/1_2-start-docker-compose.adoc
+
+        - action: execute
+          file: tutorial-steps/dev/wait-for-containers.sh
+          render:
+            skip: true
+
+
     - title: Initialize the project
       content:
         - action: make_file
@@ -65,7 +76,7 @@ dev:
     - title: Add the helper gradle tasks
       content:
         - action: execute
-          file: tutorial-steps/dev/4_0-helper-package.sh
+          file: tutorial-steps/dev/make-helper-package.sh
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/4_0-helper-package.adoc
 
@@ -75,7 +86,7 @@ dev:
             file: tutorials/window-final-result/kstreams/markup/dev/4_1-topic-task.adoc
 
         - action: make_file
-          file: src/main/java/io/confluent/developer/helper/SchemaPublicaition.java
+          file: src/main/java/io/confluent/developer/helper/SchemaPublication.java
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/4_2-schema-task.adoc
 
@@ -100,7 +111,7 @@ dev:
     - title: Compile and run the Kafka Streams program
       content:
         - action: execute_async
-          file: tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-project.sh
+          file: tutorial-steps/dev/run-project.sh
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/7_0-gradle-run.adoc
 
@@ -110,13 +121,12 @@ dev:
 
     - title: Produce events to the input topic
       content:
-        - action: execute
-          file: tutorial-steps/dev/run-producer.sh
+        - action: skip
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/8_0-produce-input.adoc
 
         - action: execute
-          file: tutorial-steps/dev/run-producer2.sh
+          file: tutorial-steps/dev/run-producer-execute.sh
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/8_1-produce-input.adoc
 
@@ -127,11 +137,22 @@ dev:
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/9_0-consume-output.adoc
 
-        - action: execute_async
-          file: tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-consumer.sh
+        - action: skip
           render:
             file: tutorials/window-final-result/kstreams/markup/dev/9_1-consume-output.adoc
 
+        - action: execute_async
+          file: tutorial-steps/dev/run-consumer-execute.sh
+          stdout: tutorial-steps/dev/outputs/actual-count.txt
+          render:
+            skip: true
+
+        - name: wait for the consumer to read the messages
+          action: sleep
+          ms: 3000
+          render:
+            skip: true
+
         - action: skip
           file: tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-consumer.sh
           render:
@@ -141,23 +162,30 @@ test:
   steps:
     - title: Create a test configuration file
       content:
+        - action: execute
+          file: tutorial-steps/test/make-test-config-dir.sh
+          render:
+            file: tutorials/window-final-result/kstreams/markup/test/1_0-make-test-config-dir.adoc
+
         - action: make_file
-          file: code/src/test/resources/test.properties
+          file: src/test/resources/test.properties
           render:
-            file: tutorials/window-final-result/kstreams/markup/test/1_0-make-test-conf.adoc
+            file: tutorials/window-final-result/kstreams/markup/test/1_1-make-test-conf.adoc
 
     - title: Write a test
       content:
         - action: execute
           file: tutorial-steps/test/make-test-dir.sh
           render:
             file: tutorials/window-final-result/kstreams/markup/test/2_0-make-test-dir.adoc
+
         - action: make_file
-          file: code/src/test/java/io/confluent/developer/WindowFinalResultTest.java
+          file: src/test/java/io/confluent/developer/WindowFinalResultTest.java
           render:
             file: tutorials/window-final-result/kstreams/markup/test/2_1-make-test-class.adoc
+
         - action: make_file
-          file: code/src/test/java/io/confluent/developer/PressureDatetimeExtractor.java
+          file: src/test/java/io/confluent/developer/PressureDatetimeExtractorTest.java
           render:
             file: tutorials/window-final-result/kstreams/markup/test/2_2-make-test-class-ts.adoc
 
@@ -171,10 +199,15 @@ prod:
   steps:
     - title: Create a production configuration file
       content:
+        - action: execute
+          file: tutorial-steps/prod/make-configuration-dir.sh
+          render:
+            file: tutorials/window-final-result/kstreams/markup/prod/1_0-make-prod-conf-dir.adoc
+
         - action: make_file
           file: configuration/prod.properties
           render:
-            file: tutorials/window-final-result/kstreams/markup/prod/1_0-make-prod-conf.adoc
+            file: tutorials/window-final-result/kstreams/markup/prod/1_1-make-prod-conf.adoc
 
     - title: Build a Docker image
       content:

_includes/tutorials/window-final-result/kstreams/code/Makefile

@@ -0,0 +1,9 @@
+STEPS_DIR := tutorial-steps
+DEV_OUTPUTS_DIR := $(STEPS_DIR)/dev/outputs
+TEMP_DIR := $(shell mktemp -d)
+
+tutorial:
+	rm -r $(DEV_OUTPUTS_DIR) || true
+	mkdir $(DEV_OUTPUTS_DIR)
+	harness-runner ../../../../../_data/harnesses/window-final-result/kstreams.yml $(TEMP_DIR)
+	bash -c 'diff --strip-trailing-cr <(cat $(STEPS_DIR)/dev/expected-count.txt) <(cat $(DEV_OUTPUTS_DIR)/actual-count.txt)'

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/docker-compose-up.sh

@@ -1 +1 @@
-docker-compose up -d
+docker-compose up

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/expected-count.txt

@@ -0,0 +1,4 @@
+5
+1
+1
+2

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-consumer-execute.sh

@@ -0,0 +1,3 @@
+docker exec -it broker /usr/bin/kafka-console-consumer --topic output-topic --bootstrap-server broker:9092\
+ --property value.deserializer=org.apache.kafka.common.serialization.LongDeserializer\
+ --from-beginning | grep -v "org.apache.kafka"

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer-alias.sh

@@ -0,0 +1,2 @@
+set +m
+function produce () { echo $1 | docker exec -i schema-registry /usr/bin/kafka-avro-console-producer --broker-list broker:9092 --topic input-topic --property value.schema="$(< src/main/avro/pressure-alert.avsc)" & }

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer-execute.sh

@@ -0,0 +1,18 @@
+# 1. Here we have to pint the event times to ensure the same result
+# 2. The produce use here doesn't run in background like in the tutorial
+
+function produce () { echo $1 | docker exec -i schema-registry /usr/bin/kafka-avro-console-producer --broker-list broker:9092 --topic input-topic --property value.schema="$(< src/main/avro/pressure-alert.avsc)"; }
+
+produce '{"id":"XXX","datetime":"2019-09-21T05:00:00.+0200","pressure":30}'
+produce '{"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}'
+produce '{"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}'
+produce '{"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}'
+produce '{"id":"102","datetime":"2019-09-21T05:45:02.+0200","pressure":30}'
+produce '{"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}'
+produce '{"id":"101","datetime":"2019-09-21T05:45:07.+0200","pressure":30}'
+produce '{"id":"101","datetime":"2019-09-21T05:44:13.+0200","pressure":30}'
+produce '{"id":"102","datetime":"2019-09-21T05:45:13.+0200","pressure":30}'
+produce '{"id":"102","datetime":"2019-09-21T05:43:23.+0200","pressure":30}'
+produce '{"id":"301","datetime":"2019-09-21T12:45:23.+0900","pressure":30}'
+produce '{"id":"301","datetime":"2019-09-21T12:45:24.+0900","pressure":30}'
+produce '{"id":"XXX","datetime":"2019-09-21T06:00:00.+0200","pressure":30}'

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer.sh

@@ -1,2 +1,16 @@
-set +m
-function produce () { echo $1 | docker exec -i schema-registry /usr/bin/kafka-avro-console-producer --broker-list broker:9092 --topic input-topic --property value.schema="$(< src/main/avro/pressure-alert.avsc)" & }
+produce '{"id":"XXX","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+produce '{"id":"101","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+produce '{"id":"101","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+produce '{"id":"101","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+produce '{"id":"102","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+sleep 10
+produce '{"id":"101","datetime":"'$(date -v-10S +%FT%T.%z)'","pressure":30}' # late of 10 sec
+produce '{"id":"101","datetime":"'$(date -v-15S +%FT%T.%z)'","pressure":30}' # late of 15 sec
+produce '{"id":"101","datetime":"'$(date -v-60S +%FT%T.%z)'","pressure":30}' # late of 01 min
+produce '{"id":"102","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+sleep 10
+produce '{"id":"102","datetime":"'$(date -v-60S +%FT%T.%z)'","pressure":30}' # out of the grace period
+export TZ=Asia/Tokyo
+produce '{"id":"301","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+produce '{"id":"301","datetime":"'$(date +%FT%T.%z)'","pressure":30}'
+produce '{"id":"XXX","datetime":"'$(date +%FT%T.%z)'","pressure":30}'

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/dev/wait-for-containers.sh

@@ -0,0 +1,6 @@
+# Wait for Schema Registry to become available
+while [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8081/) -eq 000 ]
+do
+  echo -e $(date) "Schema Registry HTTP state: " $(curl -s -o /dev/null -w %{http_code} http://localhost:8081/) " (waiting for 200)"
+  sleep 5
+done

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/prod/launch-container-conf-file.sh

@@ -1 +1 @@
-docker run -v $PWD/configuration/prod.properties:/config.properties io.confluent.developer/kstreams-filter:0.0.1-SNAPSHOT -Dconfig.file=config.properties
+docker run -v $PWD/configuration/prod.properties:/config.properties io.confluent.developer/kstreams-window-final-result:0.0.1-SNAPSHOT -Dconfig.file=/config.properties

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/prod/launch-container-env-var.sh

@@ -1,4 +1,4 @@
 docker run\
  -e "BOOTSTRAP_SERVERS=broker:9092"\
  -e "SCHEMA_REGISTRY_URL=http://schema-registry:8081"\
- io.confluent.developer/kstreams-filter:0.0.1-SNAPSHOT
+ io.confluent.developer/kstreams-window-final-result:0.0.1-SNAPSHOT

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/prod/make-configuration-dir.sh

@@ -0,0 +1 @@
+mkdir configuration

_includes/tutorials/window-final-result/kstreams/code/tutorial-steps/test/make-test-config-dir.sh

@@ -0,0 +1 @@
+mkdir -p src/test/resources

_includes/tutorials/window-final-result/kstreams/markup/dev/1_1-make-docker-compose.adoc

@@ -5,11 +5,3 @@ Next, create the following `docker-compose.yml` file to obtain Confluent Platfor
     include_raw tutorials/window-final-result/kstreams/code/docker-compose.yml
 %}</code></pre>
 +++++
-
-And launch it by running:
-
-+++++
-<pre class="snippet"><code class="shell">{%
-    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/dev/docker-compose-up.sh
-%}</code></pre>
-+++++

_includes/tutorials/window-final-result/kstreams/markup/dev/1_2-start-docker-compose.adoc

@@ -0,0 +1,5 @@
+And launch it by running:
+
++++++
+<pre class="snippet"><code class="shell">{% include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/dev/docker-compose-up.sh %}</code></pre>
++++++

_includes/tutorials/window-final-result/kstreams/markup/dev/8_0-produce-input.adoc

@@ -4,7 +4,7 @@ In a new terminal, define a produce function and tail the content of temporary f
 
 +++++
 <pre class="snippet"><code class="bash">{%
-    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer.sh
+    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer-alias.sh
 %}</code></pre>
 +++++
 

_includes/tutorials/window-final-result/kstreams/markup/dev/8_1-produce-input.adoc

@@ -2,7 +2,7 @@ Send multiple events
 +++++
 
 <pre class="snippet"><code class="bash">{%
-    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer2.sh
+    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/dev/run-producer.sh
 %}</code></pre>
 +++++
 
@@ -16,17 +16,18 @@ You may also consume the input topic to see what are the sensors dates:
 For example:
 
 +++++
-<pre class="snippet"><code class="bash"># {"id":"101","datetime":"2019-09-19T01:14:26.+0200","pressure":30}
+<pre class="snippet"><code class="bash"># {"id":"XXX","datetime":"2019-09-21T05:00:00.+0200","pressure":30}
+# ...
 # {"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}
 # {"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}
 # {"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}
 # {"id":"102","datetime":"2019-09-21T05:45:02.+0200","pressure":30}
 # {"id":"101","datetime":"2019-09-21T05:45:02.+0200","pressure":30}
-# {"id":"101","datetime":"2019-09-21T05:45:00.+0200","pressure":30} # late
-# {"id":"101","datetime":"2019-09-21T05:44:59.+0200","pressure":30} # out of time
-# {"id":"102","datetime":"2019-09-21T05:45:12.+0200","pressure":30} # new window
-# {"id":"102","datetime":"2019-09-21T05:44:52.+0200","pressure":30} # out of time
-# {"id":"301","datetime":"2019-09-21T12:45:27.+0900","pressure":30} # different time zone
-# {"id":"301","datetime":"2019-09-21T12:45:27.+0900","pressure":30} # different time zone
-# {"id":"000","datetime":"2019-09-21T05:46:03.+0200","pressure":30}</code></pre>
+# {"id":"101","datetime":"2019-09-21T05:45:07.+0200","pressure":30} # late
+# {"id":"101","datetime":"2019-09-21T05:44:13.+0200","pressure":30} # out of time
+# {"id":"102","datetime":"2019-09-21T05:45:13.+0200","pressure":30} # new window
+# {"id":"102","datetime":"2019-09-21T05:43:23.+0200","pressure":30} # out of time
+# {"id":"301","datetime":"2019-09-21T12:45:23.+0900","pressure":30} # different time zone
+# {"id":"301","datetime":"2019-09-21T12:45:24.+0900","pressure":30} # different time zone
+# {"id":"XXX","datetime":"2019-09-21T06:00:00.+0200","pressure":30}</code></pre>
 +++++

_includes/tutorials/window-final-result/kstreams/markup/prod/1_0-make-prod-conf-dir.adoc

@@ -0,0 +1,7 @@
+First, create a directory for the configuration:
+
++++++
+<pre class="snippet"><code class="bash">{%
+    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/prod/make-configuration-dir.sh
+%}</code></pre>
++++++

_includes/tutorials/window-final-result/kstreams/markup/prod/1_0-make-prod-conf.adoc → _includes/tutorials/window-final-result/kstreams/markup/prod/1_1-make-prod-conf.adoc

@@ -1,4 +1,4 @@
-First, create a new configuration file at `configuration/prod.properties` with the following content. Be sure to fill in the addresses of your production hosts and change any other parameters that make sense for your setup.
+Then, create a new configuration file at `configuration/prod.properties` with the following content. Be sure to fill in the addresses of your production hosts and change any other parameters that make sense for your setup.
 
 +++++
 <pre class="snippet"><code class="properties">{%

_includes/tutorials/window-final-result/kstreams/markup/test/1_0-make-test-config-dir.adoc

@@ -0,0 +1,7 @@
+First, create a directory for the test configuration:
+
++++++
+<pre class="snippet"><code class="bash">{%
+    include_raw tutorials/window-final-result/kstreams/code/tutorial-steps/test/make-test-config-dir.sh
+%}</code></pre>
++++++

_includes/tutorials/window-final-result/kstreams/markup/test/1_0-make-test-conf.adoc → _includes/tutorials/window-final-result/kstreams/markup/test/1_1-make-test-conf.adoc

@@ -1,4 +1,4 @@
-First, create a test file configuration named `test.properties` at `src/test/resources`:
+Then, create a test file configuration named `test.properties` at `src/test/resources`:
 
 +++++
 <pre class="snippet"><code class="properties">{%