Merge branch 'master' into suppress-buffer

confluentinc · Aug 9, 2020 · af5ba9d · af5ba9d
2 parents fed21ae + cce2c10
commit af5ba9d
Show file tree

Hide file tree

Showing 156 changed files with 6,751 additions and 569 deletions.
diff --git a/design-proposals/README.md b/design-proposals/README.md
@@ -67,9 +67,10 @@ Next KLIP number: **32**
 | [KLIP-24: KEY column semantics in queries](klip-24-key-column-semantics-in-queries.md)               | Merged         | 0.10.0            | 6.0.0      |               |
 | [KLIP-25: Removal of `WITH(KEY)` syntax](klip-25-removal-of-with-key-syntax.md)                      | Merged         | 0.10.0            | 6.0.0      |               |
 | [KLIP-26: Java client interfaces](klip-26-java-client-interfaces.md)                                 | Merged         | 0.10.0            | 6.0.0      |               |
-| [KLIP-27: Enhanced UDF Configuration Options](klip-27-enhanced-udf-configuration-options.md)                                                          | Proposal       |                   |            | [Discussion](https://github.com/confluentinc/ksql/pull/5269) |
-| KLIP-28: Introduce 'CREATE OR REPLACE' for Query Upgrades                                            | Proposal       |                   |            |               |
+| [KLIP-27: Enhanced UDF Configuration Options](klip-27-enhanced-udf-configuration-options.md)         | Proposal       |                   |            | [Discussion](https://github.com/confluentinc/ksql/pull/5269) |
+| [KLIP-28: Introduce 'CREATE OR REPLACE' for Query Upgrades](klip-28-create-or-replace.md)            | Approved       |  0.12.0           |            | [Discussion](https://github.com/confluentinc/ksql/pull/5611)              |
 | [KLIP-29: Explicit Table Primary Keys and Key-less Streams]( klip-29-explicit-keys.md)               | Merged         | 0.10.0            | 6.0.0      | [Discussion](https://github.com/confluentinc/ksql/pull/5530) |
-| KLIP-30: Lambda Functions               | Proposal         |             |       | N/A |
-| KLIP-31: Metastore Backups              | Proposal         |             |       | N/A |
+| KLIP-30: Lambda Functions                                                                            | Proposal         |             |       | [Discussion](https://github.com/confluentinc/ksql/pull/5661) |
+| [KLIP-31: Metastore Backups](klip-31-metastore-backups.md)              | Merged |  0.11.0           |  6.0.0     | [Discussion](https://github.com/confluentinc/ksql/pull/5741)|
+| [KLIP-32: SQL-based testing tool](klip-32-sql-testing-tool.md)                                      | Proposal | | | |
 
diff --git a/design-proposals/klip-32-sql-testing-tool.md b/design-proposals/klip-32-sql-testing-tool.md
diff --git a/docs/concepts/queries/push.md b/docs/concepts/queries/push.md
@@ -38,7 +38,8 @@ CREATE TABLE AS SELECT or CREATE STREAM AS SELECT statement.
 Example push query
 ==================
 
-Specify a push query by using the EMIT CHANGES clause in a SELECT statement.
+Specify a push query by using an EMIT refinement clause in a SELECT statement.
+
 The following statement shows how to select five events from a `pageviews`
 stream.  
 

diff --git a/docs/concepts/time-and-windows-in-ksqldb-queries.md b/docs/concepts/time-and-windows-in-ksqldb-queries.md
@@ -423,3 +423,22 @@ CREATE TABLE pageviews_per_region AS
 
 Note that the specified retention period should be larger than the sum of window size and any grace
 period.
+
+### Window Final Results
+
+In ksqlDB, windowed aggregations update their results continuously. As new data arrives for 
+a window, freshly computed results are emitted downstream. For many applications, this is ideal, 
+since fresh results are always available, and ksqlDB is designed to make programming 
+continuous computations seamless. However, some applications need to take action only on the final 
+result of a windowed computation. Common examples of this are sending alerts or delivering results 
+to a system that doesn’t support updates.
+
+Suppose that you have an hourly windowed count of events per user. If you want to send an alert 
+when a user has less than three events in an hour, you have a real challenge. All users would match 
+this condition at first, until they accrue enough events, so you can’t simply send an alert when 
+someone matches the condition; you have to wait until you know you won’t see any more events for a 
+particular window, and then send the alert.
+
+ksqlDB offers a clean way to define this logic: after defining your windowed aggregation, 
+you can suppress the intermediate results, emitting the final count for each user when the window 
+is closed.
diff --git a/docs/developer-guide/ksqldb-reference/create-table-as-select.md b/docs/developer-guide/ksqldb-reference/create-table-as-select.md
@@ -22,7 +22,7 @@ CREATE TABLE table_name
   [ WHERE condition ]
   [ GROUP BY grouping_expression ]
   [ HAVING having_expression ]
-  EMIT CHANGES;
+  [ EMIT output_refinement ];
 ```
 
 Description
@@ -34,6 +34,10 @@ stream the result of the query as a changelog into the topic.
 Note that the WINDOW clause can only be used if the `from_item` is a stream and the query contains
 a `GROUP BY` clause.
 
+Note that EMIT `output_refinement` defaults to `CHANGES` unless explicitly set to `FINAL` on a
+
+windowed aggregation.
+
 Joins to streams can use any stream column. If the join criteria is not the key column of the stream
 ksqlDB will internally repartition the data. 
 

diff --git a/docs/developer-guide/ksqldb-reference/drop-type.md b/docs/developer-guide/ksqldb-reference/drop-type.md
@@ -13,7 +13,7 @@ Synopsis
 --------
 
 ```sql
-DROP TYPE <type_name> AS <type>;
+DROP TYPE [IF EXISTS] <type_name> AS <type>;
 ```
 
 Description
@@ -26,6 +26,9 @@ can drop a type any time, and old queries continue to work. Also, old queries
 running with a dropped type and don't change if you register a new type with
 the same name.
 
+If the IF EXISTS clause is present, the statement doesn't fail if the
+type doesn't exist.
+
 Example
 -------
 

diff --git a/docs/developer-guide/ksqldb-reference/index.md b/docs/developer-guide/ksqldb-reference/index.md
@@ -47,7 +47,7 @@ keywords: ksqldb, api, reference, function, operator, metadata, connector, query
 ## Custom Types
 
 - [CREATE TYPE](create-type.md)
-- [DROP TYPE](drop-table.md)
+- [DROP TYPE](drop-type.md)
 - [SHOW TYPES](show-types.md)
 
 ## Metadata

diff --git a/docs/developer-guide/ksqldb-reference/operators.md b/docs/developer-guide/ksqldb-reference/operators.md
@@ -83,7 +83,7 @@ The subscript operator (`[subscript_expr]`) is used to
 reference the value at an array index or a map key.
 
 ```sql
-SELECT USERID, NICKNAMES[0] FROM USERS EMIT CHANGES;
+SELECT USERID, NICKNAMES[1] FROM USERS EMIT CHANGES;
 ```
 
 STRUCT dereference

diff --git a/docs/developer-guide/ksqldb-reference/quick-reference.md b/docs/developer-guide/ksqldb-reference/quick-reference.md
@@ -243,12 +243,12 @@ Remove a type alias from ksqlDB. For more information, see
 [DROP TYPE](../../ksqldb-reference/drop-type).
 
 ```sql
-DROP TYPE <type_name> AS <type>;
+DROP TYPE [IF EXISTS] <type_name> AS <type>;
 ```
 
 ## EMIT CHANGES
-Specify a push query in a SELECT statement. For more information, see
-[Push Queries](../../concepts/queries/push).
+Specify a push query with a continuous output refinement in a SELECT statement. 
+For more information, see [Push Queries](../../concepts/queries/push).
 
 ```sql
 CREATE STREAM stream_name
@@ -257,6 +257,20 @@ CREATE STREAM stream_name
   EMIT CHANGES;
 ```
 
+## EMIT FINAL
+Specify a push query with a suppressed output refinement in a SELECT statement on a 
+windowed aggregation. 
+For more information, see [Push Queries](../../concepts/queries/push).
+
+```sql
+CREATE TABLE table_name
+  AS SELECT  select_expr_with_aggregation [, ...]
+  FROM from_stream
+  [ WINDOW window_expression ]
+  [ GROUP BY grouping_expression ]
+  EMIT FINAL;
+```
+
 ## EXPLAIN
 Show the execution plan for a SQL expression or running query. For more
 information, see [EXPLAIN](../../ksqldb-reference/explain).

diff --git a/docs/developer-guide/ksqldb-reference/select-push-query.md b/docs/developer-guide/ksqldb-reference/select-push-query.md
@@ -20,7 +20,7 @@ SELECT select_expr [, ...]
   [ WHERE condition ]
   [ GROUP BY grouping_expression ]
   [ HAVING having_expression ]
-  EMIT CHANGES
+  EMIT [ output_refinement ]
   [ LIMIT count ];
 ```
 
@@ -184,3 +184,21 @@ SELECT windowstart, windowend, item_id, SUM(quantity)
   GROUP BY item_id
   EMIT CHANGES;
 ```
+
+#### EMIT
+
+The EMIT clause lets you control the output refinement of your push query. The output refinement is
+how you would like to *emit* your results. 
+
+ksqlDB supports the following output refinement types.
+
+#### CHANGES
+
+This is the standard output refinement for push queries, for when we would like to see all changes 
+happening.
+
+#### FINAL
+
+This is for when we want to emit only the final result of a windowed aggregation, and suppress the 
+intermediate results until the window closes. Note that this output refinement is supported only 
+for windowed aggregations.
diff --git a/docs/developer-guide/syntax-reference.md b/docs/developer-guide/syntax-reference.md
@@ -206,8 +206,8 @@ ksqlDB supports fields that are arrays of another type. All the elements
 in the array must be of the same type. The element type can be any valid
 SQL type.
 
-The elements of an array are zero-indexed and can be accessed by using
-the `[]` operator passing in the index. For example, `SOME_ARRAY[0]`
+The elements of an array are one-indexed and can be accessed by using
+the `[]` operator passing in the index. For example, `SOME_ARRAY[1]`
 retrieves the first element from the array. For more information, see
 [Operators](ksqldb-reference/operators.md).
 

diff --git a/docs/operate-and-deploy/changelog.md b/docs/operate-and-deploy/changelog.md
@@ -6,6 +6,11 @@ description: Lists changes to the ksqlDB codebase
 keywords: ksqldb, changelog
 ---
 
+Version 0.11.0
+--------------
+
+- [ksqlDB v0.11.0 changelog](https://github.com/confluentinc/ksql/blob/master/CHANGELOG.md#0110-2020-08-03)
+
 Version 0.10.1
 --------------
 

diff --git a/docs/tutorials/event-driven-microservice.md b/docs/tutorials/event-driven-microservice.md
@@ -177,7 +177,7 @@ INSERT INTO transactions (
     'derek@example.com',
     '352642227248344',
     '0cf100ca-993c-427f-9ea5-e892ef350363',
-    '2020-04-25T12:50:30',
+    '2020-04-22T12:50:30',
     18.97
 );
 
@@ -187,7 +187,7 @@ INSERT INTO transactions (
     'colin@example.com',
     '373913272311617',
     'de9831c0-7cf1-4ebf-881d-0415edec0d6b',
-    '2020-04-19T09:45:15',
+    '2020-04-22T09:45:15',
     12.50
 );
 
@@ -207,7 +207,7 @@ INSERT INTO transactions (
     'derek@example.com',
     '352642227248344',
     '5d916e65-1af3-4142-9fd3-302dd55c512f',
-    '2020-04-25T12:50:25',
+    '2020-04-22T12:50:25',
     3200.80
 );
 
@@ -217,7 +217,7 @@ INSERT INTO transactions (
     'derek@example.com',
     '352642227248344',
     'd7d47fdb-75e9-46c0-93f6-d42ff1432eea',
-    '2020-04-25T12:51:55',
+    '2020-04-22T12:51:55',
     154.32
 );
 
@@ -237,7 +237,7 @@ INSERT INTO transactions (
     'colin@example.com',
     '373913272311617',
     '2360d53e-3fad-4e9a-b306-b166b7ca4f64',
-    '2020-04-19T09:45:35',
+    '2020-04-22T09:45:35',
     234.65
 );
 
@@ -247,7 +247,7 @@ INSERT INTO transactions (
     'colin@example.com',
     '373913272311617',
     'de9831c0-7cf1-4ebf-881d-0415edec0d6b',
-    '2020-04-19T09:44:03',
+    '2020-04-22T09:44:03',
     150.00
 );
 ```

diff --git a/docs/tutorials/examples.md b/docs/tutorials/examples.md
@@ -345,7 +345,7 @@ zipcode for each user:
 
 ```sql
 CREATE STREAM pageviews_interest_contact AS
-  SELECT interests[0] AS first_interest,
+  SELECT interests[1] AS first_interest,
          contactinfo['zipcode'] AS zipcode,
          contactinfo['city'] AS city,
          viewtime,

diff --git a/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/ClientTest.java b/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/ClientTest.java
@@ -899,9 +899,24 @@ public void shouldFailToDescribeSourceViaExecuteStatement() {
     // Given
     final SourceDescriptionEntity entity = new SourceDescriptionEntity(
         "describe source;",
-        new SourceDescription("name", Optional.empty(), Collections.emptyList(), Collections.emptyList(),
-            Collections.emptyList(), "type", "timestamp", "statistics", "errorStats",
-            false, "keyFormat", "valueFormat", "topic", 4, 1, "statement"),
+        new SourceDescription(
+            "name",
+            Optional.empty(),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            "type",
+            "timestamp",
+            "statistics",
+            "errorStats",
+            false,
+            "keyFormat",
+            "valueFormat",
+            "topic",
+            4,
+            1,
+            "statement",
+            Collections.emptyList()),
         Collections.emptyList());
     testEndpoints.setKsqlEndpointResponse(Collections.singletonList(entity));