From 44015f9137ca9c9ee7d60b1d4cfe72248a769815 Mon Sep 17 00:00:00 2001 From: Wade Waldron Date: Tue, 10 Sep 2024 09:59:27 -0600 Subject: [PATCH] Initial public release of the course. (#1) * Initial Commit. * Updates to use version 0.129.0 of the Confluent Plugin. * Remove Instructions from the Readme. * Update the Changelog. --- .gitignore | 5 +- CHANGELOG.md | 6 +- README.md | 59 ++- build.sh | 59 +++ exercises/exercise.sh | 110 ++++++ .../01-connecting-to-confluent-cloud/pom.xml | 252 ++++++++++++ .../src/main/java/marketplace/.gitignore | 0 .../main/java/marketplace/Marketplace.java | 23 ++ .../main/resources/cloud-template.properties | 38 ++ .../src/main/resources/log4j2.properties | 25 ++ solutions/02-querying-flink-tables/pom.xml | 252 ++++++++++++ .../java/marketplace/CustomerService.java | 35 ++ .../main/java/marketplace/Marketplace.java | 37 ++ .../main/java/marketplace/OrderService.java | 43 +++ .../main/resources/cloud-template.properties | 38 ++ .../src/main/resources/log4j2.properties | 25 ++ .../java/marketplace/CustomerBuilder.java | 59 +++ .../CustomerServiceIntegrationTest.java | 114 ++++++ .../java/marketplace/CustomerServiceTest.java | 67 ++++ .../marketplace/FlinkIntegrationTest.java | 189 +++++++++ .../test/java/marketplace/OrderBuilder.java | 52 +++ .../OrderServiceIntegrationTest.java | 147 +++++++ .../java/marketplace/OrderServiceTest.java | 94 +++++ .../03-building-a-streaming-pipeline/pom.xml | 252 ++++++++++++ .../java/marketplace/CustomerService.java | 35 ++ .../main/java/marketplace/Marketplace.java | 41 ++ .../main/java/marketplace/OrderService.java | 76 ++++ .../main/resources/cloud-template.properties | 38 ++ .../src/main/resources/log4j2.properties | 25 ++ .../java/marketplace/CustomerBuilder.java | 59 +++ .../CustomerServiceIntegrationTest.java | 114 ++++++ .../java/marketplace/CustomerServiceTest.java | 67 ++++ .../marketplace/FlinkIntegrationTest.java | 189 +++++++++ .../test/java/marketplace/OrderBuilder.java | 52 +++ .../OrderServiceIntegrationTest.java | 254 ++++++++++++ .../java/marketplace/OrderServiceTest.java | 145 +++++++ solutions/04-windowing/pom.xml | 252 ++++++++++++ .../java/marketplace/CustomerService.java | 35 ++ .../main/java/marketplace/Marketplace.java | 45 +++ .../main/java/marketplace/OrderService.java | 117 ++++++ .../main/resources/cloud-template.properties | 38 ++ .../src/main/resources/log4j2.properties | 25 ++ .../java/marketplace/CustomerBuilder.java | 59 +++ .../CustomerServiceIntegrationTest.java | 114 ++++++ .../java/marketplace/CustomerServiceTest.java | 67 ++++ .../marketplace/FlinkIntegrationTest.java | 189 +++++++++ .../test/java/marketplace/OrderBuilder.java | 52 +++ .../OrderServiceIntegrationTest.java | 365 ++++++++++++++++++ .../java/marketplace/OrderServiceTest.java | 215 +++++++++++ solutions/05-joins/pom.xml | 252 ++++++++++++ .../main/java/marketplace/ClickService.java | 86 +++++ .../java/marketplace/CustomerService.java | 35 ++ .../main/java/marketplace/Marketplace.java | 54 +++ .../main/java/marketplace/OrderService.java | 117 ++++++ .../main/resources/cloud-template.properties | 38 ++ .../src/main/resources/log4j2.properties | 25 ++ .../test/java/marketplace/ClickBuilder.java | 61 +++ .../ClickServiceIntegrationTest.java | 241 ++++++++++++ .../java/marketplace/ClickServiceTest.java | 123 ++++++ .../java/marketplace/CustomerBuilder.java | 59 +++ .../CustomerServiceIntegrationTest.java | 114 ++++++ .../java/marketplace/CustomerServiceTest.java | 67 ++++ .../marketplace/FlinkIntegrationTest.java | 189 +++++++++ .../test/java/marketplace/OrderBuilder.java | 52 +++ .../OrderServiceIntegrationTest.java | 365 ++++++++++++++++++ .../java/marketplace/OrderServiceTest.java | 215 +++++++++++ .../01-connecting-to-confluent-cloud/pom.xml | 252 ++++++++++++ .../main/java/marketplace/Marketplace.java | 17 + .../main/resources/cloud-template.properties | 38 ++ .../src/main/resources/log4j2.properties | 25 ++ .../java/marketplace/CustomerService.java | 29 ++ .../main/java/marketplace/OrderService.java | 31 ++ .../java/marketplace/CustomerBuilder.java | 59 +++ .../CustomerServiceIntegrationTest.java | 114 ++++++ .../java/marketplace/CustomerServiceTest.java | 67 ++++ .../marketplace/FlinkIntegrationTest.java | 189 +++++++++ .../test/java/marketplace/OrderBuilder.java | 52 +++ .../OrderServiceIntegrationTest.java | 147 +++++++ .../java/marketplace/OrderServiceTest.java | 94 +++++ .../OrderServiceIntegrationTest.java | 254 ++++++++++++ .../java/marketplace/OrderServiceTest.java | 145 +++++++ .../OrderServiceIntegrationTest.java | 365 ++++++++++++++++++ .../java/marketplace/OrderServiceTest.java | 215 +++++++++++ staging/05-joins/pom.xml | 258 +++++++++++++ .../src/main/java/marketplace/.gitignore | 0 .../main/java/marketplace/ClickService.java | 38 ++ .../test/java/marketplace/ClickBuilder.java | 61 +++ .../ClickServiceIntegrationTest.java | 241 ++++++++++++ .../java/marketplace/ClickServiceTest.java | 121 ++++++ 89 files changed, 9549 insertions(+), 5 deletions(-) create mode 100755 build.sh create mode 100755 exercises/exercise.sh create mode 100644 solutions/01-connecting-to-confluent-cloud/pom.xml create mode 100644 solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/.gitignore create mode 100644 solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java create mode 100644 solutions/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties create mode 100644 solutions/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties create mode 100644 solutions/02-querying-flink-tables/pom.xml create mode 100644 solutions/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java create mode 100644 solutions/02-querying-flink-tables/src/main/java/marketplace/Marketplace.java create mode 100644 solutions/02-querying-flink-tables/src/main/java/marketplace/OrderService.java create mode 100644 solutions/02-querying-flink-tables/src/main/resources/cloud-template.properties create mode 100644 solutions/02-querying-flink-tables/src/main/resources/log4j2.properties create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java create mode 100644 solutions/03-building-a-streaming-pipeline/pom.xml create mode 100644 solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/CustomerService.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/Marketplace.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/OrderService.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/main/resources/cloud-template.properties create mode 100644 solutions/03-building-a-streaming-pipeline/src/main/resources/log4j2.properties create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerBuilder.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceIntegrationTest.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceTest.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/FlinkIntegrationTest.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderBuilder.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java create mode 100644 solutions/04-windowing/pom.xml create mode 100644 solutions/04-windowing/src/main/java/marketplace/CustomerService.java create mode 100644 solutions/04-windowing/src/main/java/marketplace/Marketplace.java create mode 100644 solutions/04-windowing/src/main/java/marketplace/OrderService.java create mode 100644 solutions/04-windowing/src/main/resources/cloud-template.properties create mode 100644 solutions/04-windowing/src/main/resources/log4j2.properties create mode 100644 solutions/04-windowing/src/test/java/marketplace/CustomerBuilder.java create mode 100644 solutions/04-windowing/src/test/java/marketplace/CustomerServiceIntegrationTest.java create mode 100644 solutions/04-windowing/src/test/java/marketplace/CustomerServiceTest.java create mode 100644 solutions/04-windowing/src/test/java/marketplace/FlinkIntegrationTest.java create mode 100644 solutions/04-windowing/src/test/java/marketplace/OrderBuilder.java create mode 100644 solutions/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 solutions/04-windowing/src/test/java/marketplace/OrderServiceTest.java create mode 100644 solutions/05-joins/pom.xml create mode 100644 solutions/05-joins/src/main/java/marketplace/ClickService.java create mode 100644 solutions/05-joins/src/main/java/marketplace/CustomerService.java create mode 100644 solutions/05-joins/src/main/java/marketplace/Marketplace.java create mode 100644 solutions/05-joins/src/main/java/marketplace/OrderService.java create mode 100644 solutions/05-joins/src/main/resources/cloud-template.properties create mode 100644 solutions/05-joins/src/main/resources/log4j2.properties create mode 100644 solutions/05-joins/src/test/java/marketplace/ClickBuilder.java create mode 100644 solutions/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java create mode 100644 solutions/05-joins/src/test/java/marketplace/ClickServiceTest.java create mode 100644 solutions/05-joins/src/test/java/marketplace/CustomerBuilder.java create mode 100644 solutions/05-joins/src/test/java/marketplace/CustomerServiceIntegrationTest.java create mode 100644 solutions/05-joins/src/test/java/marketplace/CustomerServiceTest.java create mode 100644 solutions/05-joins/src/test/java/marketplace/FlinkIntegrationTest.java create mode 100644 solutions/05-joins/src/test/java/marketplace/OrderBuilder.java create mode 100644 solutions/05-joins/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 solutions/05-joins/src/test/java/marketplace/OrderServiceTest.java create mode 100644 staging/01-connecting-to-confluent-cloud/pom.xml create mode 100644 staging/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java create mode 100644 staging/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties create mode 100644 staging/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties create mode 100644 staging/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java create mode 100644 staging/02-querying-flink-tables/src/main/java/marketplace/OrderService.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java create mode 100644 staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java create mode 100644 staging/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java create mode 100644 staging/04-windowing/src/test/java/marketplace/OrderServiceTest.java create mode 100644 staging/05-joins/pom.xml create mode 100644 staging/05-joins/src/main/java/marketplace/.gitignore create mode 100644 staging/05-joins/src/main/java/marketplace/ClickService.java create mode 100644 staging/05-joins/src/test/java/marketplace/ClickBuilder.java create mode 100644 staging/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java create mode 100644 staging/05-joins/src/test/java/marketplace/ClickServiceTest.java diff --git a/.gitignore b/.gitignore index 3daf146..e070529 100644 --- a/.gitignore +++ b/.gitignore @@ -24,13 +24,12 @@ target # Properties files (may contain user credentials) -consumer.properties -producer.properties +cloud.properties # Exercises folder exercises/* !exercises/exercise.sh -!exercises/exercise.bat +!exercises/README.md /flink* diff --git a/CHANGELOG.md b/CHANGELOG.md index 45ebef8..e522881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,8 @@ ## Version 0.1.0 -* Initial commit of required files for a public repo. \ No newline at end of file +* Initial commit of required files for a public repo. + +## Version 0.2.0 + +* Initial commit of the exercise code. \ No newline at end of file diff --git a/README.md b/README.md index 13e1a33..5ea8dce 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,60 @@ # Apache Flink Table API for Java -This repository is for the **Apache Flink Table API for Java** course provided by Confluent Developer. \ No newline at end of file +This repository is for the **Apache Flink Table API for Java** course provided by Confluent Developer. + +## A Simple Marketplace with Apache Flink + +Throughout this course, we'll be executing a series of Flink queries focusing on an eCommerce Marketplace. Before you get started with the exercises, take a moment to familiarize yourself with the repository. + +## Exercises + +The course is broken down into several exercises. You will work on exercises in the `./exercises` folder. + +### exercises/exercise.sh + +The `exercise.sh` script is there to help you advance through the exercises. + +The basic flow of an exercise is: + +- `stage` the exercise (I.E. Import any new code necessary to begin the exercise). +- `solve` the exercise (Either manually, or using the `solve` command below). + +You can list the exercises by running: + +```bash +./exercise.sh list +``` + +You can stage an exercise by running: + +```bash +./exercise.sh stage +``` + +You can automatically solve an exercise by running: + +```bash +./exercise.sh solve +``` + +**WARNING:** Solving an exercise will overwrite your code. + +You can solve a single file by running: + +```bash +./exercise.sh solve +``` + +**NOTE:** We encourage you to solve the exercise yourself. If you get stuck, you can always look at the solution in the `solutions` folder (see below). + +## Staging + +The `staging` folder contains the files necessary to set up each exercise. These will be copied to the `exercises` folder when you execute the `stage` command with the `exercise.sh` script. + +In general, you can ignore this folder. + +## Solutions + +The `solutions` folder contains complete solutions for each exercise. These will be copied to the `exercises` folder when you execute the `solve` command with the `exercise.sh` script. + +In general, you can ignore this folder, but you might find it helpful to reference if you get stuck. \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..c012337 --- /dev/null +++ b/build.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +EXERCISES_DIR=exercises +SOLUTIONS_DIR=solutions +STAGING_DIR=staging + +function help() { + echo "Usage:" + echo " build.sh " + echo " Commands:" + echo " validate - Run through each exercise, stage the exercise, then apply the solution. Verify the exercise builds in it's final state." +} + +function validate() { + WORKING_DIR=$(pwd) + + EXERCISES=($(ls $SOLUTIONS_DIR/ | grep "^[0-9]*")) + + TMP_DIR=target/tmp + rm -rf $TMP_DIR + mkdir -p $TMP_DIR + + cp -r $EXERCISES_DIR $TMP_DIR + cp -r $SOLUTIONS_DIR $TMP_DIR + cp -r $STAGING_DIR $TMP_DIR + + cd $TMP_DIR/$EXERCISES_DIR + + for EXERCISE in "${EXERCISES[@]}" + do + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo $EXERCISE + ./exercise.sh stage $EXERCISE + ./exercise.sh solve $EXERCISE + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + + if [ -f "pom.xml" ]; then + mvn clean test + fi + done + + rm -rf $TMP_DIR + + cd $WORKING_DIR +} + +## Determine which command is being requested, and execute it. +COMMAND=${1:-"help"} +if [ "$COMMAND" = "validate" ]; then + validate +elif [ "$COMMAND" = "help" ]; then + help +else + echo "INVALID COMMAND: $COMMAND" + help + exit 1 +fi \ No newline at end of file diff --git a/exercises/exercise.sh b/exercises/exercise.sh new file mode 100755 index 0000000..2a9b137 --- /dev/null +++ b/exercises/exercise.sh @@ -0,0 +1,110 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +EXERCISE_DIR=./ +SOLUTIONS_DIR=../solutions +STAGING_DIR=../staging + +if [ ! -d $STAGING_DIR ]; then + echo "$STAGING_DIR could not be found." + exit 1 +fi + +if [ ! -d $SOLUTIONS_DIR ]; then + echo "$SOLUTIONS_DIR could not be found." + exit 1 +fi + +function help() { + echo "Usage:" + echo " exercises.sh " + echo " Commands:" + echo " stage - Setup the exercise." + echo " - A portion of the exercise name (eg. the exercise number) that will be used to select the exercise." + echo " solve - Solve the exercise." + echo " - A portion of the exercise name (eg. the exercise number) that will be used to select the exercise." + echo " - (Optional) A portion of a file name that will be used to select while file to copy from the solution." + echo " list - List all exercises." + echo " Exercise Filter: A portion of the name of the exercise. Eg. The Exercise Number. If multiple matches are found, the first one will be chosen." +} + +function stage() { + EXERCISE_FILTER=$1 + MATCHED_EXERCISES=($(ls $STAGING_DIR | grep ".*$EXERCISE_FILTER.*")) + EXERCISE=${MATCHED_EXERCISES[0]} + + echo "STAGING $EXERCISE" + + cp -r $STAGING_DIR/$EXERCISE/. $EXERCISE_DIR +} + +function solve() { + EXERCISE_FILTER=$1 + FILE_FILTER=${2:-""} + MATCHED_EXERCISES=($(ls $SOLUTIONS_DIR | grep ".*$EXERCISE_FILTER.*")) + EXERCISE=${MATCHED_EXERCISES[0]} + SOLUTION=$SOLUTIONS_DIR/$EXERCISE + + if [ -z $FILE_FILTER ]; then + echo "SOLVING $EXERCISE" + + cp -r $SOLUTION/. $EXERCISE_DIR + else + WORKING_DIR=$(pwd) + cd $SOLUTION + MATCHED_FILES=($(find . -iname "*$FILE_FILTER*")) + cd $WORKING_DIR + + if [ -z ${MATCHED_FILES:-""} ]; then + echo "FILE NOT FOUND: $FILE_FILTER" + exit 1 + fi + + FILE_PATH=${MATCHED_FILES[0]} + + echo "COPYING $FILE_PATH FROM $EXERCISE" + + cp $SOLUTION/$FILE_PATH $EXERCISE_DIR/$FILE_PATH + fi + +} + +function list() { + EXERCISES=$(ls $SOLUTIONS_DIR) + + for ex in "${EXERCISES[@]}" + do + echo "$ex" + done +} + +COMMAND=${1:-"help"} + +## Determine which command is being requested, and execute it. +if [ "$COMMAND" = "stage" ]; then + EXERCISE_FILTER=${2:-""} + if [ -z $EXERCISE_FILTER ]; then + echo "MISSING EXERCISE ID" + help + exit 1 + fi + stage $EXERCISE_FILTER +elif [ "$COMMAND" = "solve" ]; then + EXERCISE_FILTER=${2:-""} + FILE_FILTER=${3:-""} + if [ -z $EXERCISE_FILTER ]; then + echo "MISSING EXERCISE ID" + help + exit 1 + fi + solve $EXERCISE_FILTER $FILE_FILTER +elif [ "$COMMAND" = "list" ]; then + list +elif [ "$COMMAND" = "help" ]; then + help +else + echo "INVALID COMMAND: $COMMAND" + help + exit 1 +fi \ No newline at end of file diff --git a/solutions/01-connecting-to-confluent-cloud/pom.xml b/solutions/01-connecting-to-confluent-cloud/pom.xml new file mode 100644 index 0000000..2654828 --- /dev/null +++ b/solutions/01-connecting-to-confluent-cloud/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.129.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + 5.11.0 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-flink-table-api-java-plugin + ${confluent-plugin.version} + + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + + + + 5 + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/.gitignore b/solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java b/solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java new file mode 100644 index 0000000..71fa35c --- /dev/null +++ b/solutions/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java @@ -0,0 +1,23 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; + +import java.io.File; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; + +public class Marketplace { + + public static void main(String[] args) throws Exception { + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + TableEnvironment env = TableEnvironment.create(settings); + + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + Arrays.stream(env.listTables()).forEach(System.out::println); + } +} diff --git a/solutions/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties b/solutions/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties new file mode 100644 index 0000000..528e4c2 --- /dev/null +++ b/solutions/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties @@ -0,0 +1,38 @@ +##################################################################### +# Confluent Cloud Connection Configuration # +# # +# Note: The plugin supports different ways of passing parameters: # +# - Programmatically # +# - Via global environment variables # +# - Via arguments in the main() method # +# - Via properties file # +# # +# For all cases, use the ConfluentSettings class to get started. # +# # +# The project is preconfigured with this properties file. # +##################################################################### + +# Cloud region +client.cloud= +client.region= + +# Access & compute resources +client.flink-api-key= +client.flink-api-secret= +client.organization-id= +client.environment-id= +client.compute-pool-id= + +# User or service account +client.principal-id= + +# Kafka (used by tests) +client.kafka.bootstrap.servers= +client.kafka.security.protocol=SASL_SSL +client.kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='' password=''; +client.kafka.sasl.mechanism=PLAIN + +# Schema Registry (used by tests) +client.registry.url= +client.registry.key= +client.registry.secret= \ No newline at end of file diff --git a/solutions/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties b/solutions/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties new file mode 100644 index 0000000..32c696e --- /dev/null +++ b/solutions/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/solutions/02-querying-flink-tables/pom.xml b/solutions/02-querying-flink-tables/pom.xml new file mode 100644 index 0000000..2654828 --- /dev/null +++ b/solutions/02-querying-flink-tables/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.129.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + 5.11.0 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-flink-table-api-java-plugin + ${confluent-plugin.version} + + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + + + + 5 + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/solutions/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java b/solutions/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java new file mode 100644 index 0000000..55b3ced --- /dev/null +++ b/solutions/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java @@ -0,0 +1,35 @@ +package marketplace; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import static org.apache.flink.table.api.Expressions.$; + +public class CustomerService { + private final TableEnvironment env; + private final String customersTableName; + + public CustomerService( + TableEnvironment env, + String customersTableName + ) { + this.env = env; + this.customersTableName = customersTableName; + } + + public TableResult allCustomers() { + return env.from(customersTableName) + .select($("*")) + .execute(); + } + + public TableResult allCustomerAddresses() { + return env.from(customersTableName) + .select( + $("customer_id"), + $("address"), + $("postcode"), + $("city") + ).execute(); + } +} diff --git a/solutions/02-querying-flink-tables/src/main/java/marketplace/Marketplace.java b/solutions/02-querying-flink-tables/src/main/java/marketplace/Marketplace.java new file mode 100644 index 0000000..a7ebea9 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/main/java/marketplace/Marketplace.java @@ -0,0 +1,37 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; + +import java.io.File; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; + +public class Marketplace { + + public static void main(String[] args) throws Exception { + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + TableEnvironment env = TableEnvironment.create(settings); + + CustomerService customers = new CustomerService( + env, + "`examples`.`marketplace`.`customers`" + ); + OrderService orders = new OrderService( + env, + "`examples`.`marketplace`.`orders`" + ); + + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + Arrays.stream(env.listTables()).forEach(System.out::println); + + customers.allCustomers(); + customers.allCustomerAddresses(); + orders.ordersOver50Dollars(); + orders.pricesWithTax(BigDecimal.valueOf(1.1)); + } +} diff --git a/solutions/02-querying-flink-tables/src/main/java/marketplace/OrderService.java b/solutions/02-querying-flink-tables/src/main/java/marketplace/OrderService.java new file mode 100644 index 0000000..0b3ec9a --- /dev/null +++ b/solutions/02-querying-flink-tables/src/main/java/marketplace/OrderService.java @@ -0,0 +1,43 @@ +package marketplace; + +import org.apache.flink.table.api.*; + +import java.math.BigDecimal; +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class OrderService { + private final TableEnvironment env; + private final String ordersTableName; + + public OrderService( + TableEnvironment env, + String ordersTableName + ) { + this.env = env; + this.ordersTableName = ordersTableName; + } + + public TableResult ordersOver50Dollars() { + return env.from(ordersTableName) + .select($("*")) + .where($("price").isGreaterOrEqual(50)) + .execute(); + } + + public TableResult pricesWithTax(BigDecimal taxAmount) { + return env.from(ordersTableName) + .select( + $("order_id"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .as("original_price"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .times(taxAmount) + .round(2) + .as("price_with_tax") + ).execute(); + } +} diff --git a/solutions/02-querying-flink-tables/src/main/resources/cloud-template.properties b/solutions/02-querying-flink-tables/src/main/resources/cloud-template.properties new file mode 100644 index 0000000..528e4c2 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/main/resources/cloud-template.properties @@ -0,0 +1,38 @@ +##################################################################### +# Confluent Cloud Connection Configuration # +# # +# Note: The plugin supports different ways of passing parameters: # +# - Programmatically # +# - Via global environment variables # +# - Via arguments in the main() method # +# - Via properties file # +# # +# For all cases, use the ConfluentSettings class to get started. # +# # +# The project is preconfigured with this properties file. # +##################################################################### + +# Cloud region +client.cloud= +client.region= + +# Access & compute resources +client.flink-api-key= +client.flink-api-secret= +client.organization-id= +client.environment-id= +client.compute-pool-id= + +# User or service account +client.principal-id= + +# Kafka (used by tests) +client.kafka.bootstrap.servers= +client.kafka.security.protocol=SASL_SSL +client.kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='' password=''; +client.kafka.sasl.mechanism=PLAIN + +# Schema Registry (used by tests) +client.registry.url= +client.registry.key= +client.registry.secret= \ No newline at end of file diff --git a/solutions/02-querying-flink-tables/src/main/resources/log4j2.properties b/solutions/02-querying-flink-tables/src/main/resources/log4j2.properties new file mode 100644 index 0000000..32c696e --- /dev/null +++ b/solutions/02-querying-flink-tables/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java new file mode 100644 index 0000000..5bbf318 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java @@ -0,0 +1,59 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.util.Random; + +class CustomerBuilder { + private int customerId; + private String name; + private String address; + private String postCode; + private String city; + private String email; + + private final Random rnd = new Random(System.currentTimeMillis()); + + public CustomerBuilder() { + customerId = rnd.nextInt(1000); + name = "Name" + rnd.nextInt(1000); + address = "Address" + rnd.nextInt(1000); + postCode = "PostCode" + rnd.nextInt(1000); + city = "City" + rnd.nextInt(1000); + email = "Email" + rnd.nextInt(1000); + } + + public CustomerBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public CustomerBuilder withName(String name) { + this.name = name; + return this; + } + + public CustomerBuilder withAddress(String address) { + this.address = address; + return this; + } + + public CustomerBuilder withPostCode(String postCode) { + this.postCode = postCode; + return this; + } + + public CustomerBuilder withCity(String city) { + this.city = city; + return this; + } + + public CustomerBuilder withEmail(String email) { + this.email = email; + return this; + } + + public Row build() { + return Row.of(customerId, name, address, postCode, city, email); + } +} diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java new file mode 100644 index 0000000..34a4d28 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java @@ -0,0 +1,114 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class CustomerServiceIntegrationTest extends FlinkIntegrationTest { + private final String customersTableName = "`flink-table-api-java`.`marketplace`.`customers-temp`"; + + private final String customersTableDefinition = + "CREATE TABLE " + customersTableName + " (\n" + + " `customer_id` INT NOT NULL,\n" + + " `name` VARCHAR(2147483647) NOT NULL,\n" + + " `address` VARCHAR(2147483647) NOT NULL,\n" + + " `postcode` VARCHAR(2147483647) NOT NULL,\n" + + " `city` VARCHAR(2147483647) NOT NULL,\n" + + " `email` VARCHAR(2147483647) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private CustomerService customerService; + + @Override + public void setup() { + customerService = new CustomerService( + env, + customersTableName + ); + } + + @Test + @Timeout(90) + public void allCustomers_shouldReturnTheDetailsOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomers()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(customers), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id","name", "address", "postcode", "city", "email" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } + + @Test + @Timeout(90) + public void allCustomerAddresses_shouldReturnTheAddressesOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomerAddresses()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(customers.size(), actual.size()); + + List expected = customers.stream() + .map(row -> Row.project(row, new int[] {0, 2, 3, 4})) + .toList(); + + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id", "address", "postcode", "city" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } +} \ No newline at end of file diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java new file mode 100644 index 0000000..081b1d0 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java @@ -0,0 +1,67 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@Tag("UnitTest") +public class CustomerServiceTest { + + private CustomerService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockResult = mock(TableResult.class); + service = new CustomerService(mockEnv, "customerTable"); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.execute()).thenReturn(mockResult); + } + + @Test + public void allCustomers_shouldSelectAllFields() { + TableResult result = service.allCustomers(); + + verify(mockEnv).from("customerTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + assertEquals(mockResult, result); + } + + @Test + public void allCustomerAddresses_shouldSelectOnlyTheRelevantFields() { + TableResult result = service.allCustomerAddresses(); + + verify(mockEnv).from("customerTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] {"customer_id", "address", "postcode", "city"}, selectArgs.toArray()); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java new file mode 100644 index 0000000..f919c37 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java @@ -0,0 +1,189 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.common.KafkaFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class FlinkIntegrationTest { + private List jobsToCancel; + private List topicsToDelete; + private Thread shutdownHook; + private boolean isShuttingDown; + + protected TableEnvironment env; + protected AdminClient adminClient; + protected SchemaRegistryClient registryClient; + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected void setup() {}; + protected void teardown() {}; + + @BeforeEach + public void mainSetup() throws Exception { + jobsToCancel = new ArrayList<>(); + topicsToDelete = new ArrayList<>(); + + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + env = TableEnvironment.create(settings); + + Properties properties = new Properties(); + settings.getConfiguration().toMap().forEach((k,v) -> + properties.put(k.replace("client.kafka.", ""), v) + ); + adminClient = AdminClient.create(properties); + + Map schemaConfig = new HashMap<>(); + + schemaConfig.put("basic.auth.credentials.source", "USER_INFO"); + schemaConfig.put( + "basic.auth.user.info", + properties.get("client.registry.key") + ":" + properties.get("client.registry.secret")); + + registryClient = new CachedSchemaRegistryClient( + properties.getProperty("client.registry.url"), + 100, + schemaConfig + ); + + isShuttingDown = false; + shutdownHook = new Thread(() -> { + logger.info("Shutdown Detected. Cleaning up resources."); + isShuttingDown = true; + mainTeardown(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + setup(); + } + + @AfterEach + public void mainTeardown() { + teardown(); + + jobsToCancel.forEach(result -> + result.getJobClient() + .orElseThrow() + .cancel() + .join() + ); + + topicsToDelete.forEach(topic -> + deleteTopic(topic) + ); + + if(!isShuttingDown) { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } + } + + protected TableResult retry(Supplier supplier) { + return retry(3, supplier); + } + + protected TableResult retry(int tries, Supplier supplier) { + try { + return supplier.get(); + } catch (Exception e) { + logger.error("Failed on retryable command.", e); + + if(tries > 0) { + logger.info("Retrying"); + return retry(tries - 1, supplier); + } else { + logger.info("Maximum number of tries exceeded. Failing..."); + throw e; + } + } + } + + protected TableResult cancelOnExit(TableResult tableResult) { + jobsToCancel.add(tableResult); + return tableResult; + } + + protected Stream fetchRows(TableResult result) { + Iterable iterable = result::collect; + return StreamSupport.stream(iterable.spliterator(), false); + } + + protected String getShortTableName(String tableName) { + String[] tablePath = tableName.split("\\."); + return tablePath[tablePath.length - 1].replace("`",""); + } + + protected void deleteTopic(String topicName) { + try { + String schemaName = topicName + "-value"; + logger.info("Deleting Schema: "+schemaName); + if(registryClient.getAllSubjects().contains(schemaName)) { + registryClient.deleteSubject(schemaName, false); + registryClient.deleteSubject(schemaName, true); + } + logger.info("Deleted Schema: "+schemaName); + } catch (Exception e) { + logger.error("Error Deleting Schema", e); + } + + try { + if(adminClient.listTopics().names().get().contains(topicName)) { + logger.info("Deleting Topic: " + topicName); + KafkaFuture result = adminClient.deleteTopics(List.of(topicName)).all(); + + while(!result.isDone()) { + logger.info("Waiting for topic to be deleted: " + topicName); + Thread.sleep(1000); + } + + logger.info("Topic Deleted: " + topicName); + } + } catch (Exception e) { + logger.error("Error Deleting Topic", e); + } + } + + protected void deleteTable(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopic(topicName); + } + + protected void deleteTopicOnExit(String topicName) { + topicsToDelete.add(topicName); + } + + protected void deleteTableOnExit(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopicOnExit(topicName); + } + + protected void createTemporaryTable(String fullyQualifiedTableName, String tableDefinition) { + String topicName = getShortTableName(fullyQualifiedTableName); + + logger.info("Creating temporary table: " + fullyQualifiedTableName); + + try { + env.executeSql(tableDefinition).await(); + deleteTopicOnExit(topicName); + + logger.info("Created temporary table: " + fullyQualifiedTableName); + } catch (Exception e) { + logger.error("Unable to create temporary table: " + fullyQualifiedTableName, e); + } + } +} diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java new file mode 100644 index 0000000..1b5e2e6 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java @@ -0,0 +1,52 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +class OrderBuilder { + private String orderId, productId; + private Integer customerId; + private Double price; + private Instant timestamp; + + public OrderBuilder() { + Random rnd = new Random(); + orderId = "Order" + rnd.nextInt(1000); + customerId = rnd.nextInt(1000); + productId = "Product" + rnd.nextInt(1000); + price = rnd.nextDouble(100); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public OrderBuilder withOrderId(String orderId) { + this.orderId = orderId; + return this; + } + + public OrderBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public OrderBuilder withProductId(String productId) { + this.productId = productId; + return this; + } + + public OrderBuilder withPrice(Double price) { + this.price = price; + return this; + } + + public OrderBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(orderId, customerId, productId, price, timestamp); + } +} diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..f9d619c --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,147 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } +} \ No newline at end of file diff --git a/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java b/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..6788739 --- /dev/null +++ b/solutions/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,94 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/pom.xml b/solutions/03-building-a-streaming-pipeline/pom.xml new file mode 100644 index 0000000..2654828 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.129.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + 5.11.0 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-flink-table-api-java-plugin + ${confluent-plugin.version} + + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + + + + 5 + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/CustomerService.java b/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/CustomerService.java new file mode 100644 index 0000000..55b3ced --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/CustomerService.java @@ -0,0 +1,35 @@ +package marketplace; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import static org.apache.flink.table.api.Expressions.$; + +public class CustomerService { + private final TableEnvironment env; + private final String customersTableName; + + public CustomerService( + TableEnvironment env, + String customersTableName + ) { + this.env = env; + this.customersTableName = customersTableName; + } + + public TableResult allCustomers() { + return env.from(customersTableName) + .select($("*")) + .execute(); + } + + public TableResult allCustomerAddresses() { + return env.from(customersTableName) + .select( + $("customer_id"), + $("address"), + $("postcode"), + $("city") + ).execute(); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/Marketplace.java b/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/Marketplace.java new file mode 100644 index 0000000..01c6af8 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/Marketplace.java @@ -0,0 +1,41 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; + +import java.io.File; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; + +public class Marketplace { + + public static void main(String[] args) throws Exception { + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + TableEnvironment env = TableEnvironment.create(settings); + + CustomerService customers = new CustomerService( + env, + "`examples`.`marketplace`.`customers`" + ); + OrderService orders = new OrderService( + env, + "`examples`.`marketplace`.`orders`", + "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping`" + ); + + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + Arrays.stream(env.listTables()).forEach(System.out::println); + + customers.allCustomers(); + customers.allCustomerAddresses(); + orders.ordersOver50Dollars(); + orders.pricesWithTax(BigDecimal.valueOf(1.1)); + + orders.createFreeShippingTable(); + orders.streamOrdersOver50Dollars(); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/OrderService.java b/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/OrderService.java new file mode 100644 index 0000000..2741d77 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/main/java/marketplace/OrderService.java @@ -0,0 +1,76 @@ +package marketplace; + +import org.apache.flink.table.api.*; + +import java.math.BigDecimal; +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class OrderService { + private final TableEnvironment env; + private final String ordersTableName; + private final String freeShippingTableName; + + public OrderService( + TableEnvironment env, + String ordersTableName, + String freeShippingTableName + ) { + this.env = env; + this.ordersTableName = ordersTableName; + this.freeShippingTableName = freeShippingTableName; + } + + public TableResult createFreeShippingTable() { + return env.executeSql( + "CREATE TABLE IF NOT EXISTS "+freeShippingTableName+" (\n" + + " `order_id` STRING NOT NULL,\n" + + " `details` ROW (\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` STRING NOT NULL,\n" + + " `price` DOUBLE NOT NULL \n" + + " ) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"); + } + + public TableResult ordersOver50Dollars() { + return env.from(ordersTableName) + .select($("*")) + .where($("price").isGreaterOrEqual(50)) + .execute(); + } + + public TableResult streamOrdersOver50Dollars() { + return env.from(ordersTableName) + .where($("price").isGreaterOrEqual(50)) + .select( + $("order_id"), + row( + $("customer_id"), + $("product_id"), + $("price") + ).as("details") + ) + .insertInto(freeShippingTableName) + .execute(); + } + + public TableResult pricesWithTax(BigDecimal taxAmount) { + return env.from(ordersTableName) + .select( + $("order_id"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .as("original_price"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .times(taxAmount) + .round(2) + .as("price_with_tax") + ).execute(); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/main/resources/cloud-template.properties b/solutions/03-building-a-streaming-pipeline/src/main/resources/cloud-template.properties new file mode 100644 index 0000000..528e4c2 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/main/resources/cloud-template.properties @@ -0,0 +1,38 @@ +##################################################################### +# Confluent Cloud Connection Configuration # +# # +# Note: The plugin supports different ways of passing parameters: # +# - Programmatically # +# - Via global environment variables # +# - Via arguments in the main() method # +# - Via properties file # +# # +# For all cases, use the ConfluentSettings class to get started. # +# # +# The project is preconfigured with this properties file. # +##################################################################### + +# Cloud region +client.cloud= +client.region= + +# Access & compute resources +client.flink-api-key= +client.flink-api-secret= +client.organization-id= +client.environment-id= +client.compute-pool-id= + +# User or service account +client.principal-id= + +# Kafka (used by tests) +client.kafka.bootstrap.servers= +client.kafka.security.protocol=SASL_SSL +client.kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='' password=''; +client.kafka.sasl.mechanism=PLAIN + +# Schema Registry (used by tests) +client.registry.url= +client.registry.key= +client.registry.secret= \ No newline at end of file diff --git a/solutions/03-building-a-streaming-pipeline/src/main/resources/log4j2.properties b/solutions/03-building-a-streaming-pipeline/src/main/resources/log4j2.properties new file mode 100644 index 0000000..32c696e --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerBuilder.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerBuilder.java new file mode 100644 index 0000000..5bbf318 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerBuilder.java @@ -0,0 +1,59 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.util.Random; + +class CustomerBuilder { + private int customerId; + private String name; + private String address; + private String postCode; + private String city; + private String email; + + private final Random rnd = new Random(System.currentTimeMillis()); + + public CustomerBuilder() { + customerId = rnd.nextInt(1000); + name = "Name" + rnd.nextInt(1000); + address = "Address" + rnd.nextInt(1000); + postCode = "PostCode" + rnd.nextInt(1000); + city = "City" + rnd.nextInt(1000); + email = "Email" + rnd.nextInt(1000); + } + + public CustomerBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public CustomerBuilder withName(String name) { + this.name = name; + return this; + } + + public CustomerBuilder withAddress(String address) { + this.address = address; + return this; + } + + public CustomerBuilder withPostCode(String postCode) { + this.postCode = postCode; + return this; + } + + public CustomerBuilder withCity(String city) { + this.city = city; + return this; + } + + public CustomerBuilder withEmail(String email) { + this.email = email; + return this; + } + + public Row build() { + return Row.of(customerId, name, address, postCode, city, email); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceIntegrationTest.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceIntegrationTest.java new file mode 100644 index 0000000..34a4d28 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceIntegrationTest.java @@ -0,0 +1,114 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class CustomerServiceIntegrationTest extends FlinkIntegrationTest { + private final String customersTableName = "`flink-table-api-java`.`marketplace`.`customers-temp`"; + + private final String customersTableDefinition = + "CREATE TABLE " + customersTableName + " (\n" + + " `customer_id` INT NOT NULL,\n" + + " `name` VARCHAR(2147483647) NOT NULL,\n" + + " `address` VARCHAR(2147483647) NOT NULL,\n" + + " `postcode` VARCHAR(2147483647) NOT NULL,\n" + + " `city` VARCHAR(2147483647) NOT NULL,\n" + + " `email` VARCHAR(2147483647) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private CustomerService customerService; + + @Override + public void setup() { + customerService = new CustomerService( + env, + customersTableName + ); + } + + @Test + @Timeout(90) + public void allCustomers_shouldReturnTheDetailsOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomers()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(customers), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id","name", "address", "postcode", "city", "email" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } + + @Test + @Timeout(90) + public void allCustomerAddresses_shouldReturnTheAddressesOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomerAddresses()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(customers.size(), actual.size()); + + List expected = customers.stream() + .map(row -> Row.project(row, new int[] {0, 2, 3, 4})) + .toList(); + + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id", "address", "postcode", "city" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } +} \ No newline at end of file diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceTest.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceTest.java new file mode 100644 index 0000000..081b1d0 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/CustomerServiceTest.java @@ -0,0 +1,67 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@Tag("UnitTest") +public class CustomerServiceTest { + + private CustomerService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockResult = mock(TableResult.class); + service = new CustomerService(mockEnv, "customerTable"); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.execute()).thenReturn(mockResult); + } + + @Test + public void allCustomers_shouldSelectAllFields() { + TableResult result = service.allCustomers(); + + verify(mockEnv).from("customerTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + assertEquals(mockResult, result); + } + + @Test + public void allCustomerAddresses_shouldSelectOnlyTheRelevantFields() { + TableResult result = service.allCustomerAddresses(); + + verify(mockEnv).from("customerTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] {"customer_id", "address", "postcode", "city"}, selectArgs.toArray()); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/FlinkIntegrationTest.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/FlinkIntegrationTest.java new file mode 100644 index 0000000..f919c37 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/FlinkIntegrationTest.java @@ -0,0 +1,189 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.common.KafkaFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class FlinkIntegrationTest { + private List jobsToCancel; + private List topicsToDelete; + private Thread shutdownHook; + private boolean isShuttingDown; + + protected TableEnvironment env; + protected AdminClient adminClient; + protected SchemaRegistryClient registryClient; + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected void setup() {}; + protected void teardown() {}; + + @BeforeEach + public void mainSetup() throws Exception { + jobsToCancel = new ArrayList<>(); + topicsToDelete = new ArrayList<>(); + + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + env = TableEnvironment.create(settings); + + Properties properties = new Properties(); + settings.getConfiguration().toMap().forEach((k,v) -> + properties.put(k.replace("client.kafka.", ""), v) + ); + adminClient = AdminClient.create(properties); + + Map schemaConfig = new HashMap<>(); + + schemaConfig.put("basic.auth.credentials.source", "USER_INFO"); + schemaConfig.put( + "basic.auth.user.info", + properties.get("client.registry.key") + ":" + properties.get("client.registry.secret")); + + registryClient = new CachedSchemaRegistryClient( + properties.getProperty("client.registry.url"), + 100, + schemaConfig + ); + + isShuttingDown = false; + shutdownHook = new Thread(() -> { + logger.info("Shutdown Detected. Cleaning up resources."); + isShuttingDown = true; + mainTeardown(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + setup(); + } + + @AfterEach + public void mainTeardown() { + teardown(); + + jobsToCancel.forEach(result -> + result.getJobClient() + .orElseThrow() + .cancel() + .join() + ); + + topicsToDelete.forEach(topic -> + deleteTopic(topic) + ); + + if(!isShuttingDown) { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } + } + + protected TableResult retry(Supplier supplier) { + return retry(3, supplier); + } + + protected TableResult retry(int tries, Supplier supplier) { + try { + return supplier.get(); + } catch (Exception e) { + logger.error("Failed on retryable command.", e); + + if(tries > 0) { + logger.info("Retrying"); + return retry(tries - 1, supplier); + } else { + logger.info("Maximum number of tries exceeded. Failing..."); + throw e; + } + } + } + + protected TableResult cancelOnExit(TableResult tableResult) { + jobsToCancel.add(tableResult); + return tableResult; + } + + protected Stream fetchRows(TableResult result) { + Iterable iterable = result::collect; + return StreamSupport.stream(iterable.spliterator(), false); + } + + protected String getShortTableName(String tableName) { + String[] tablePath = tableName.split("\\."); + return tablePath[tablePath.length - 1].replace("`",""); + } + + protected void deleteTopic(String topicName) { + try { + String schemaName = topicName + "-value"; + logger.info("Deleting Schema: "+schemaName); + if(registryClient.getAllSubjects().contains(schemaName)) { + registryClient.deleteSubject(schemaName, false); + registryClient.deleteSubject(schemaName, true); + } + logger.info("Deleted Schema: "+schemaName); + } catch (Exception e) { + logger.error("Error Deleting Schema", e); + } + + try { + if(adminClient.listTopics().names().get().contains(topicName)) { + logger.info("Deleting Topic: " + topicName); + KafkaFuture result = adminClient.deleteTopics(List.of(topicName)).all(); + + while(!result.isDone()) { + logger.info("Waiting for topic to be deleted: " + topicName); + Thread.sleep(1000); + } + + logger.info("Topic Deleted: " + topicName); + } + } catch (Exception e) { + logger.error("Error Deleting Topic", e); + } + } + + protected void deleteTable(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopic(topicName); + } + + protected void deleteTopicOnExit(String topicName) { + topicsToDelete.add(topicName); + } + + protected void deleteTableOnExit(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopicOnExit(topicName); + } + + protected void createTemporaryTable(String fullyQualifiedTableName, String tableDefinition) { + String topicName = getShortTableName(fullyQualifiedTableName); + + logger.info("Creating temporary table: " + fullyQualifiedTableName); + + try { + env.executeSql(tableDefinition).await(); + deleteTopicOnExit(topicName); + + logger.info("Created temporary table: " + fullyQualifiedTableName); + } catch (Exception e) { + logger.error("Unable to create temporary table: " + fullyQualifiedTableName, e); + } + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderBuilder.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderBuilder.java new file mode 100644 index 0000000..1b5e2e6 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderBuilder.java @@ -0,0 +1,52 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +class OrderBuilder { + private String orderId, productId; + private Integer customerId; + private Double price; + private Instant timestamp; + + public OrderBuilder() { + Random rnd = new Random(); + orderId = "Order" + rnd.nextInt(1000); + customerId = rnd.nextInt(1000); + productId = "Product" + rnd.nextInt(1000); + price = rnd.nextDouble(100); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public OrderBuilder withOrderId(String orderId) { + this.orderId = orderId; + return this; + } + + public OrderBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public OrderBuilder withProductId(String productId) { + this.productId = productId; + return this; + } + + public OrderBuilder withPrice(Double price) { + this.price = price; + return this; + } + + public OrderBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(orderId, customerId, productId, price, timestamp); + } +} diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..6e9e87f --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,254 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderQualifiedForFreeShippingTableName = "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping-temp`"; + private final String orderQualifiedForFreeShippingShortTableName = "order-qualified-for-free-shipping-temp"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + private Row toQualifiedForFreeShippingRow(Row row) { + return Row.of( + row.getFieldAs(indexOf("order_id")), + Row.of( + row.getFieldAs(indexOf("customer_id")), + row.getFieldAs(indexOf("product_id")), + row.getFieldAs(indexOf("price")) + ) + ); + } + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName, + orderQualifiedForFreeShippingTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(60) + public void createFreeShippingTable_shouldCreateTheTable() { + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + TableResult result = orderService.createFreeShippingTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderQualifiedForFreeShippingShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderQualifiedForFreeShippingShortTableName), + "Could not find the table: "+orderQualifiedForFreeShippingShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderQualifiedForFreeShippingShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrdersOver50Dollars_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(orderQualifiedForFreeShippingTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the destination table. + orderService.createFreeShippingTable().await(); + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + final int detailsPosition = 1; + + // Create a list of orders with specific prices. + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Initiate the stream. + cancelOnExit(orderService.streamOrdersOver50Dollars()); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderQualifiedForFreeShippingTableName) + .select($("*")) + .execute() + ); + + // Obtain the actual results. + List actual = fetchRows(queryResult) + .limit(Arrays.stream(prices).filter(p -> p >= 50).count()) + .toList(); + + // Build the expected results + List expected = orders.stream() + .filter(row -> row.getFieldAs(indexOf("price")) >= 50) + .map(this::toQualifiedForFreeShippingRow) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + assertEquals( + new HashSet<>(Arrays.asList("order_id", "details")), + actual.getFirst().getFieldNames(true) + ); + + assertEquals( + new HashSet<>(Arrays.asList("customer_id", "product_id", "price")), + actual.getFirst().getFieldAs(detailsPosition).getFieldNames(true) + ); + } +} \ No newline at end of file diff --git a/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..7ef6e72 --- /dev/null +++ b/solutions/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,145 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable", + "freeShippingTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } + + @Test + public void createFreeShippingTable_shouldSendTheExpectedSQL() { + TableResult result = service.createFreeShippingTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("freeShippingTable")); + assertTrue(sql.contains("details")); + assertTrue(sql.toLowerCase().contains("row")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("product_id")); + assertTrue(sql.contains("price")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrdersOver50Dollars_shouldStreamTheExpectedRecordsToTheTable() { + TableResult result = service.streamOrdersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(2, expressions.length); + assertEquals("order_id", expressions[0].asSummaryString()); + assertEquals("as(row(customer_id, product_id, price), 'details')", expressions[1].asSummaryString()); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "freeShippingTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/04-windowing/pom.xml b/solutions/04-windowing/pom.xml new file mode 100644 index 0000000..2654828 --- /dev/null +++ b/solutions/04-windowing/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.129.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + 5.11.0 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-flink-table-api-java-plugin + ${confluent-plugin.version} + + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + + + + 5 + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/solutions/04-windowing/src/main/java/marketplace/CustomerService.java b/solutions/04-windowing/src/main/java/marketplace/CustomerService.java new file mode 100644 index 0000000..55b3ced --- /dev/null +++ b/solutions/04-windowing/src/main/java/marketplace/CustomerService.java @@ -0,0 +1,35 @@ +package marketplace; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import static org.apache.flink.table.api.Expressions.$; + +public class CustomerService { + private final TableEnvironment env; + private final String customersTableName; + + public CustomerService( + TableEnvironment env, + String customersTableName + ) { + this.env = env; + this.customersTableName = customersTableName; + } + + public TableResult allCustomers() { + return env.from(customersTableName) + .select($("*")) + .execute(); + } + + public TableResult allCustomerAddresses() { + return env.from(customersTableName) + .select( + $("customer_id"), + $("address"), + $("postcode"), + $("city") + ).execute(); + } +} diff --git a/solutions/04-windowing/src/main/java/marketplace/Marketplace.java b/solutions/04-windowing/src/main/java/marketplace/Marketplace.java new file mode 100644 index 0000000..0326255 --- /dev/null +++ b/solutions/04-windowing/src/main/java/marketplace/Marketplace.java @@ -0,0 +1,45 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; + +import java.io.File; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; + +public class Marketplace { + + public static void main(String[] args) throws Exception { + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + TableEnvironment env = TableEnvironment.create(settings); + + CustomerService customers = new CustomerService( + env, + "`examples`.`marketplace`.`customers`" + ); + OrderService orders = new OrderService( + env, + "`examples`.`marketplace`.`orders`", + "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping`", + "`flink-table-api-java`.`marketplace`.`customer-orders-collected-for-period`" + ); + + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + Arrays.stream(env.listTables()).forEach(System.out::println); + + customers.allCustomers(); + customers.allCustomerAddresses(); + orders.ordersOver50Dollars(); + orders.pricesWithTax(BigDecimal.valueOf(1.1)); + + orders.createFreeShippingTable(); + orders.streamOrdersOver50Dollars(); + + orders.createOrdersForPeriodTable(); + orders.streamOrdersForPeriod(Duration.ofMinutes(1)); + } +} diff --git a/solutions/04-windowing/src/main/java/marketplace/OrderService.java b/solutions/04-windowing/src/main/java/marketplace/OrderService.java new file mode 100644 index 0000000..964d65e --- /dev/null +++ b/solutions/04-windowing/src/main/java/marketplace/OrderService.java @@ -0,0 +1,117 @@ +package marketplace; + +import org.apache.flink.table.api.*; + +import java.math.BigDecimal; +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class OrderService { + private final TableEnvironment env; + private final String ordersTableName; + private final String freeShippingTableName; + private final String ordersForPeriodTableName; + + public OrderService( + TableEnvironment env, + String ordersTableName, + String freeShippingTableName, + String ordersForPeriodTableName + ) { + this.env = env; + this.ordersTableName = ordersTableName; + this.freeShippingTableName = freeShippingTableName; + this.ordersForPeriodTableName = ordersForPeriodTableName; + } + + public TableResult createFreeShippingTable() { + return env.executeSql( + "CREATE TABLE IF NOT EXISTS "+freeShippingTableName+" (\n" + + " `order_id` STRING NOT NULL,\n" + + " `details` ROW (\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` STRING NOT NULL,\n" + + " `price` DOUBLE NOT NULL \n" + + " ) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"); + } + + public TableResult createOrdersForPeriodTable() { + return env.executeSql( + "CREATE TABLE IF NOT EXISTS "+ordersForPeriodTableName+" (\n" + + " `customer_id` INT NOT NULL,\n" + + " `window_start` TIMESTAMP(3) NOT NULL,\n" + + " `window_end` TIMESTAMP(3) NOT NULL,\n" + + " `period_in_seconds` BIGINT NOT NULL,\n" + + " `product_ids` MULTISET NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");" + ); + } + + public TableResult ordersOver50Dollars() { + return env.from(ordersTableName) + .select($("*")) + .where($("price").isGreaterOrEqual(50)) + .execute(); + } + + public TableResult streamOrdersOver50Dollars() { + return env.from(ordersTableName) + .where($("price").isGreaterOrEqual(50)) + .select( + $("order_id"), + row( + $("customer_id"), + $("product_id"), + $("price") + ).as("details") + ) + .insertInto(freeShippingTableName) + .execute(); + } + + public TableResult pricesWithTax(BigDecimal taxAmount) { + return env.from(ordersTableName) + .select( + $("order_id"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .as("original_price"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .times(taxAmount) + .round(2) + .as("price_with_tax") + ).execute(); + } + + public TableResult streamOrdersForPeriod(Duration period) { + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + return env.from(ordersTableName) + .window( + Tumble.over(lit(period.toSeconds()).seconds()) + .on($("$rowtime")) + .as("window") + ).groupBy( + $("customer_id"),$("window") + ).select( + $("customer_id"), + $("window").start(), + $("window").end(), + lit(period.toSeconds()).seconds().as("period_in_seconds"), + $("product_id").collect().as("product_ids") + ).insertInto( + ordersForPeriodTableName + ) + .execute(); + } +} diff --git a/solutions/04-windowing/src/main/resources/cloud-template.properties b/solutions/04-windowing/src/main/resources/cloud-template.properties new file mode 100644 index 0000000..528e4c2 --- /dev/null +++ b/solutions/04-windowing/src/main/resources/cloud-template.properties @@ -0,0 +1,38 @@ +##################################################################### +# Confluent Cloud Connection Configuration # +# # +# Note: The plugin supports different ways of passing parameters: # +# - Programmatically # +# - Via global environment variables # +# - Via arguments in the main() method # +# - Via properties file # +# # +# For all cases, use the ConfluentSettings class to get started. # +# # +# The project is preconfigured with this properties file. # +##################################################################### + +# Cloud region +client.cloud= +client.region= + +# Access & compute resources +client.flink-api-key= +client.flink-api-secret= +client.organization-id= +client.environment-id= +client.compute-pool-id= + +# User or service account +client.principal-id= + +# Kafka (used by tests) +client.kafka.bootstrap.servers= +client.kafka.security.protocol=SASL_SSL +client.kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='' password=''; +client.kafka.sasl.mechanism=PLAIN + +# Schema Registry (used by tests) +client.registry.url= +client.registry.key= +client.registry.secret= \ No newline at end of file diff --git a/solutions/04-windowing/src/main/resources/log4j2.properties b/solutions/04-windowing/src/main/resources/log4j2.properties new file mode 100644 index 0000000..32c696e --- /dev/null +++ b/solutions/04-windowing/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/solutions/04-windowing/src/test/java/marketplace/CustomerBuilder.java b/solutions/04-windowing/src/test/java/marketplace/CustomerBuilder.java new file mode 100644 index 0000000..5bbf318 --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/CustomerBuilder.java @@ -0,0 +1,59 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.util.Random; + +class CustomerBuilder { + private int customerId; + private String name; + private String address; + private String postCode; + private String city; + private String email; + + private final Random rnd = new Random(System.currentTimeMillis()); + + public CustomerBuilder() { + customerId = rnd.nextInt(1000); + name = "Name" + rnd.nextInt(1000); + address = "Address" + rnd.nextInt(1000); + postCode = "PostCode" + rnd.nextInt(1000); + city = "City" + rnd.nextInt(1000); + email = "Email" + rnd.nextInt(1000); + } + + public CustomerBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public CustomerBuilder withName(String name) { + this.name = name; + return this; + } + + public CustomerBuilder withAddress(String address) { + this.address = address; + return this; + } + + public CustomerBuilder withPostCode(String postCode) { + this.postCode = postCode; + return this; + } + + public CustomerBuilder withCity(String city) { + this.city = city; + return this; + } + + public CustomerBuilder withEmail(String email) { + this.email = email; + return this; + } + + public Row build() { + return Row.of(customerId, name, address, postCode, city, email); + } +} diff --git a/solutions/04-windowing/src/test/java/marketplace/CustomerServiceIntegrationTest.java b/solutions/04-windowing/src/test/java/marketplace/CustomerServiceIntegrationTest.java new file mode 100644 index 0000000..34a4d28 --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/CustomerServiceIntegrationTest.java @@ -0,0 +1,114 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class CustomerServiceIntegrationTest extends FlinkIntegrationTest { + private final String customersTableName = "`flink-table-api-java`.`marketplace`.`customers-temp`"; + + private final String customersTableDefinition = + "CREATE TABLE " + customersTableName + " (\n" + + " `customer_id` INT NOT NULL,\n" + + " `name` VARCHAR(2147483647) NOT NULL,\n" + + " `address` VARCHAR(2147483647) NOT NULL,\n" + + " `postcode` VARCHAR(2147483647) NOT NULL,\n" + + " `city` VARCHAR(2147483647) NOT NULL,\n" + + " `email` VARCHAR(2147483647) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private CustomerService customerService; + + @Override + public void setup() { + customerService = new CustomerService( + env, + customersTableName + ); + } + + @Test + @Timeout(90) + public void allCustomers_shouldReturnTheDetailsOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomers()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(customers), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id","name", "address", "postcode", "city", "email" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } + + @Test + @Timeout(90) + public void allCustomerAddresses_shouldReturnTheAddressesOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomerAddresses()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(customers.size(), actual.size()); + + List expected = customers.stream() + .map(row -> Row.project(row, new int[] {0, 2, 3, 4})) + .toList(); + + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id", "address", "postcode", "city" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } +} \ No newline at end of file diff --git a/solutions/04-windowing/src/test/java/marketplace/CustomerServiceTest.java b/solutions/04-windowing/src/test/java/marketplace/CustomerServiceTest.java new file mode 100644 index 0000000..081b1d0 --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/CustomerServiceTest.java @@ -0,0 +1,67 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@Tag("UnitTest") +public class CustomerServiceTest { + + private CustomerService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockResult = mock(TableResult.class); + service = new CustomerService(mockEnv, "customerTable"); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.execute()).thenReturn(mockResult); + } + + @Test + public void allCustomers_shouldSelectAllFields() { + TableResult result = service.allCustomers(); + + verify(mockEnv).from("customerTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + assertEquals(mockResult, result); + } + + @Test + public void allCustomerAddresses_shouldSelectOnlyTheRelevantFields() { + TableResult result = service.allCustomerAddresses(); + + verify(mockEnv).from("customerTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] {"customer_id", "address", "postcode", "city"}, selectArgs.toArray()); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/04-windowing/src/test/java/marketplace/FlinkIntegrationTest.java b/solutions/04-windowing/src/test/java/marketplace/FlinkIntegrationTest.java new file mode 100644 index 0000000..f919c37 --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/FlinkIntegrationTest.java @@ -0,0 +1,189 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.common.KafkaFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class FlinkIntegrationTest { + private List jobsToCancel; + private List topicsToDelete; + private Thread shutdownHook; + private boolean isShuttingDown; + + protected TableEnvironment env; + protected AdminClient adminClient; + protected SchemaRegistryClient registryClient; + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected void setup() {}; + protected void teardown() {}; + + @BeforeEach + public void mainSetup() throws Exception { + jobsToCancel = new ArrayList<>(); + topicsToDelete = new ArrayList<>(); + + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + env = TableEnvironment.create(settings); + + Properties properties = new Properties(); + settings.getConfiguration().toMap().forEach((k,v) -> + properties.put(k.replace("client.kafka.", ""), v) + ); + adminClient = AdminClient.create(properties); + + Map schemaConfig = new HashMap<>(); + + schemaConfig.put("basic.auth.credentials.source", "USER_INFO"); + schemaConfig.put( + "basic.auth.user.info", + properties.get("client.registry.key") + ":" + properties.get("client.registry.secret")); + + registryClient = new CachedSchemaRegistryClient( + properties.getProperty("client.registry.url"), + 100, + schemaConfig + ); + + isShuttingDown = false; + shutdownHook = new Thread(() -> { + logger.info("Shutdown Detected. Cleaning up resources."); + isShuttingDown = true; + mainTeardown(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + setup(); + } + + @AfterEach + public void mainTeardown() { + teardown(); + + jobsToCancel.forEach(result -> + result.getJobClient() + .orElseThrow() + .cancel() + .join() + ); + + topicsToDelete.forEach(topic -> + deleteTopic(topic) + ); + + if(!isShuttingDown) { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } + } + + protected TableResult retry(Supplier supplier) { + return retry(3, supplier); + } + + protected TableResult retry(int tries, Supplier supplier) { + try { + return supplier.get(); + } catch (Exception e) { + logger.error("Failed on retryable command.", e); + + if(tries > 0) { + logger.info("Retrying"); + return retry(tries - 1, supplier); + } else { + logger.info("Maximum number of tries exceeded. Failing..."); + throw e; + } + } + } + + protected TableResult cancelOnExit(TableResult tableResult) { + jobsToCancel.add(tableResult); + return tableResult; + } + + protected Stream fetchRows(TableResult result) { + Iterable iterable = result::collect; + return StreamSupport.stream(iterable.spliterator(), false); + } + + protected String getShortTableName(String tableName) { + String[] tablePath = tableName.split("\\."); + return tablePath[tablePath.length - 1].replace("`",""); + } + + protected void deleteTopic(String topicName) { + try { + String schemaName = topicName + "-value"; + logger.info("Deleting Schema: "+schemaName); + if(registryClient.getAllSubjects().contains(schemaName)) { + registryClient.deleteSubject(schemaName, false); + registryClient.deleteSubject(schemaName, true); + } + logger.info("Deleted Schema: "+schemaName); + } catch (Exception e) { + logger.error("Error Deleting Schema", e); + } + + try { + if(adminClient.listTopics().names().get().contains(topicName)) { + logger.info("Deleting Topic: " + topicName); + KafkaFuture result = adminClient.deleteTopics(List.of(topicName)).all(); + + while(!result.isDone()) { + logger.info("Waiting for topic to be deleted: " + topicName); + Thread.sleep(1000); + } + + logger.info("Topic Deleted: " + topicName); + } + } catch (Exception e) { + logger.error("Error Deleting Topic", e); + } + } + + protected void deleteTable(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopic(topicName); + } + + protected void deleteTopicOnExit(String topicName) { + topicsToDelete.add(topicName); + } + + protected void deleteTableOnExit(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopicOnExit(topicName); + } + + protected void createTemporaryTable(String fullyQualifiedTableName, String tableDefinition) { + String topicName = getShortTableName(fullyQualifiedTableName); + + logger.info("Creating temporary table: " + fullyQualifiedTableName); + + try { + env.executeSql(tableDefinition).await(); + deleteTopicOnExit(topicName); + + logger.info("Created temporary table: " + fullyQualifiedTableName); + } catch (Exception e) { + logger.error("Unable to create temporary table: " + fullyQualifiedTableName, e); + } + } +} diff --git a/solutions/04-windowing/src/test/java/marketplace/OrderBuilder.java b/solutions/04-windowing/src/test/java/marketplace/OrderBuilder.java new file mode 100644 index 0000000..1b5e2e6 --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/OrderBuilder.java @@ -0,0 +1,52 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +class OrderBuilder { + private String orderId, productId; + private Integer customerId; + private Double price; + private Instant timestamp; + + public OrderBuilder() { + Random rnd = new Random(); + orderId = "Order" + rnd.nextInt(1000); + customerId = rnd.nextInt(1000); + productId = "Product" + rnd.nextInt(1000); + price = rnd.nextDouble(100); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public OrderBuilder withOrderId(String orderId) { + this.orderId = orderId; + return this; + } + + public OrderBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public OrderBuilder withProductId(String productId) { + this.productId = productId; + return this; + } + + public OrderBuilder withPrice(Double price) { + this.price = price; + return this; + } + + public OrderBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(orderId, customerId, productId, price, timestamp); + } +} diff --git a/solutions/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java b/solutions/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..e8c4467 --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,365 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderQualifiedForFreeShippingTableName = "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping-temp`"; + private final String orderQualifiedForFreeShippingShortTableName = "order-qualified-for-free-shipping-temp"; + private final String customerOrdersForPeriodTableName = "`flink-table-api-java`.`marketplace`.`customer-orders-collected-for-period-temp`"; + private final String customerOrdersForPeriodShortTableName = "customer-orders-collected-for-period-temp"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + private Row toQualifiedForFreeShippingRow(Row row) { + return Row.of( + row.getFieldAs(indexOf("order_id")), + Row.of( + row.getFieldAs(indexOf("customer_id")), + row.getFieldAs(indexOf("product_id")), + row.getFieldAs(indexOf("price")) + ) + ); + } + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName, + orderQualifiedForFreeShippingTableName, + customerOrdersForPeriodTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(60) + public void createFreeShippingTable_shouldCreateTheTable() { + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + TableResult result = orderService.createFreeShippingTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderQualifiedForFreeShippingShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderQualifiedForFreeShippingShortTableName), + "Could not find the table: "+orderQualifiedForFreeShippingShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderQualifiedForFreeShippingShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrdersOver50Dollars_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(orderQualifiedForFreeShippingTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the destination table. + orderService.createFreeShippingTable().await(); + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + final int detailsPosition = 1; + + // Create a list of orders with specific prices. + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Initiate the stream. + cancelOnExit(orderService.streamOrdersOver50Dollars()); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderQualifiedForFreeShippingTableName) + .select($("*")) + .execute() + ); + + // Obtain the actual results. + List actual = fetchRows(queryResult) + .limit(Arrays.stream(prices).filter(p -> p >= 50).count()) + .toList(); + + // Build the expected results + List expected = orders.stream() + .filter(row -> row.getFieldAs(indexOf("price")) >= 50) + .map(this::toQualifiedForFreeShippingRow) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + assertEquals( + new HashSet<>(Arrays.asList("order_id", "details")), + actual.getFirst().getFieldNames(true) + ); + + assertEquals( + new HashSet<>(Arrays.asList("customer_id", "product_id", "price")), + actual.getFirst().getFieldAs(detailsPosition).getFieldNames(true) + ); + } + + @Test + @Timeout(60) + public void createOrdersForPeriodTable_shouldCreateTheTable() { + deleteTableOnExit(customerOrdersForPeriodTableName); + + TableResult result = orderService.createOrdersForPeriodTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+customerOrdersForPeriodShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(customerOrdersForPeriodShortTableName), + "Could not find the table: "+customerOrdersForPeriodShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE "+customerOrdersForPeriodTableName + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamCustomerPurchasesDuringPeriod_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(customerOrdersForPeriodTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the output table. + orderService.createOrdersForPeriodTable().await(); + deleteTableOnExit(customerOrdersForPeriodTableName); + + final Duration windowSize = Duration.ofSeconds(10); + + // Create a set of products for each customer. + Map> customerProducts = new HashMap<>(); + customerProducts.put(1, Arrays.asList("Product1", "Product2")); + customerProducts.put(2, Arrays.asList("Product3", "Product4")); + customerProducts.put(3, Arrays.asList("Product5")); + customerProducts.put(4, Arrays.asList("Product6", "Product7")); + customerProducts.put(5, Arrays.asList("Product8", "Product9","Product9")); // Product9 is duplicated. + + // Create an order for each product. + List orders = customerProducts.entrySet().stream() + .flatMap(entry -> + entry.getValue().stream().map(v -> + new OrderBuilder() + .withCustomerId(entry.getKey()) + .withProductId(v) + .withTimestamp(Instant.now().truncatedTo(ChronoUnit.MILLIS).minus(windowSize)) + .build() + ) + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Flink only evaluates windows when a new event arrives. + // Therefore, in order to trigger the window closing, we need to issue + // another event after the window period. Because we are specifying the + // other events to occur 10 seconds early, just issuing an event now + // will do the trick. + env.fromValues(new OrderBuilder().build()).insertInto(ordersTableName).execute(); + + // Execute the method being tested. + cancelOnExit(orderService.streamOrdersForPeriod(windowSize)); + + // Fetch the results. + TableResult results = retry(() -> + env.from(customerOrdersForPeriodTableName) + .select($("*")) + .execute() + ); + + List actual = fetchRows(results) + .limit(customerProducts.size()).toList(); + + // Assert on the results. + actual.forEach(row -> { + Integer customerId = row.getFieldAs("customer_id"); + + Map actualProducts = row.getFieldAs("product_ids"); + Map expectedProducts = customerProducts.get(customerId).stream() + .collect(Collectors.groupingBy(product -> product, Collectors.summingInt(x -> 1))); + + assertEquals(expectedProducts, actualProducts); + assertEquals( + row.getFieldAs("window_start"), + row.getFieldAs("window_end").minus(windowSize) + ); + assertEquals(windowSize.toSeconds(), row.getFieldAs("period_in_seconds")); + }); + } +} \ No newline at end of file diff --git a/solutions/04-windowing/src/test/java/marketplace/OrderServiceTest.java b/solutions/04-windowing/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..01429cd --- /dev/null +++ b/solutions/04-windowing/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,215 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable", + "freeShippingTable", + "ordersForPeriodTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } + + @Test + public void createFreeShippingTable_shouldSendTheExpectedSQL() { + TableResult result = service.createFreeShippingTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("freeShippingTable")); + assertTrue(sql.contains("details")); + assertTrue(sql.toLowerCase().contains("row")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("product_id")); + assertTrue(sql.contains("price")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrdersOver50Dollars_shouldStreamTheExpectedRecordsToTheTable() { + TableResult result = service.streamOrdersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(2, expressions.length); + assertEquals("order_id", expressions[0].asSummaryString()); + assertEquals("as(row(customer_id, product_id, price), 'details')", expressions[1].asSummaryString()); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "freeShippingTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } + + @Test + public void createOrdersForPeriodTable_shouldSendTheExpectedSQL() { + TableResult result = service.createOrdersForPeriodTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("ordersForPeriodTable")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("window_start")); + assertTrue(sql.contains("window_end")); + assertTrue(sql.contains("period_in_seconds")); + assertTrue(sql.contains("product_ids")); + + assertEquals(mockResult, result); + } + + @Test + public void streamCustomerPurchasesDuringPeriod_shouldStreamTheExpectedRecordsToTheTable() { + Duration windowSize = Duration.ofSeconds(10); + TableResult result = service.streamOrdersForPeriod(windowSize); + + verify(mockEnv).useCatalog("examples"); + verify(mockEnv).useDatabase("marketplace"); + verify(mockEnv).from("orderTable"); + + ArgumentCaptor windowCaptor = ArgumentCaptor.forClass(GroupWindow.class); + verify(mockTable).window(windowCaptor.capture()); + + TumbleWithSizeOnTimeWithAlias window = (TumbleWithSizeOnTimeWithAlias) windowCaptor.getValue(); + String windowAlias = window.getAlias().asSummaryString(); + String timeField = window.getTimeField().asSummaryString(); + long size = Long.parseLong(window.getSize().asSummaryString()); + + assertEquals("$rowtime", timeField); + assertEquals(10000l, size); + + ArgumentCaptor groupByCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockGroupWindowedTable).groupBy(groupByCaptor.capture()); + Expression[] groupByExpressions = groupByCaptor.getValue(); + + assertEquals(2, groupByExpressions.length); + assertEquals("customer_id", groupByExpressions[0].asSummaryString()); + assertEquals(windowAlias, groupByExpressions[1].asSummaryString()); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockWindowGroupedTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(5, expressions.length); + assertEquals("customer_id", expressions[0].asSummaryString()); + assertEquals("start("+windowAlias+")", expressions[1].asSummaryString()); + assertEquals("end("+windowAlias+")", expressions[2].asSummaryString()); + assertEquals("as("+lit(windowSize.getSeconds()).seconds().asSummaryString()+", 'period_in_seconds')" , expressions[3].asSummaryString()); + assertEquals("as(collect(product_id), 'product_ids')", expressions[4].asSummaryString()); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "ordersForPeriodTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/05-joins/pom.xml b/solutions/05-joins/pom.xml new file mode 100644 index 0000000..2654828 --- /dev/null +++ b/solutions/05-joins/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.129.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + 5.11.0 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-flink-table-api-java-plugin + ${confluent-plugin.version} + + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + + + + 5 + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/solutions/05-joins/src/main/java/marketplace/ClickService.java b/solutions/05-joins/src/main/java/marketplace/ClickService.java new file mode 100644 index 0000000..e6dfa11 --- /dev/null +++ b/solutions/05-joins/src/main/java/marketplace/ClickService.java @@ -0,0 +1,86 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class ClickService { + private final TableEnvironment env; + private final String clicksTableName; + private final String ordersTableName; + private final String orderPlacedAfterClickTableName; + + public ClickService( + TableEnvironment env, + String clicksTableName, + String ordersTableName, + String orderPlacedAfterClickTableName + ) { + this.env = env; + this.clicksTableName = clicksTableName; + this.ordersTableName = ordersTableName; + this.orderPlacedAfterClickTableName = orderPlacedAfterClickTableName; + } + + public TableResult createOrderPlacedAfterClickTable() { + return env.executeSql( + "CREATE TABLE IF NOT EXISTS "+orderPlacedAfterClickTableName+" (\n" + + " `customer_id` INT NOT NULL,\n" + + " `clicked_url` VARCHAR(2147483647) NOT NULL,\n" + + " `time_of_click` TIMESTAMP_LTZ(3) NOT NULL,\n" + + " `purchased_product` VARCHAR(2147483647) NOT NULL,\n" + + " `time_of_order` TIMESTAMP_LTZ(3) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");" + ); + } + + public TableResult streamOrderPlacedAfterClick(Duration withinTimePeriod) { + createOrderPlacedAfterClickTable(); + + Table clicks = env + .from(clicksTableName) + .select( + $("user_id"), + $("url"), + $("$rowtime").as("time_of_click") + ); + + Table orders = env + .from(ordersTableName) + .select( + $("customer_id"), + $("product_id"), + $("$rowtime").as("time_of_order") + ); + + return clicks + .join(orders) + .where( + and( + $("user_id").isEqual($("customer_id")), + $("time_of_order").isGreaterOrEqual( + $("time_of_click") + ), + $("time_of_order").isLess( + $("time_of_click").plus(lit(withinTimePeriod.toSeconds()).seconds()) + ) + ) + ) + .select( + $("customer_id"), + $("url").as("clicked_url"), + $("time_of_click"), + $("product_id").as("purchased_product"), + $("time_of_order") + ) + .insertInto(orderPlacedAfterClickTableName) + .execute(); + } +} diff --git a/solutions/05-joins/src/main/java/marketplace/CustomerService.java b/solutions/05-joins/src/main/java/marketplace/CustomerService.java new file mode 100644 index 0000000..55b3ced --- /dev/null +++ b/solutions/05-joins/src/main/java/marketplace/CustomerService.java @@ -0,0 +1,35 @@ +package marketplace; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import static org.apache.flink.table.api.Expressions.$; + +public class CustomerService { + private final TableEnvironment env; + private final String customersTableName; + + public CustomerService( + TableEnvironment env, + String customersTableName + ) { + this.env = env; + this.customersTableName = customersTableName; + } + + public TableResult allCustomers() { + return env.from(customersTableName) + .select($("*")) + .execute(); + } + + public TableResult allCustomerAddresses() { + return env.from(customersTableName) + .select( + $("customer_id"), + $("address"), + $("postcode"), + $("city") + ).execute(); + } +} diff --git a/solutions/05-joins/src/main/java/marketplace/Marketplace.java b/solutions/05-joins/src/main/java/marketplace/Marketplace.java new file mode 100644 index 0000000..43d6c78 --- /dev/null +++ b/solutions/05-joins/src/main/java/marketplace/Marketplace.java @@ -0,0 +1,54 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; + +import java.io.File; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; + +public class Marketplace { + + public static void main(String[] args) throws Exception { + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + TableEnvironment env = TableEnvironment.create(settings); + + CustomerService customers = new CustomerService( + env, + "`examples`.`marketplace`.`customers`" + ); + OrderService orders = new OrderService( + env, + "`examples`.`marketplace`.`orders`", + "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping`", + "`flink-table-api-java`.`marketplace`.`customer-orders-collected-for-period`" + ); + ClickService clicks = new ClickService( + env, + "`examples`.`marketplace`.`clicks`", + "`examples`.`marketplace`.`orders`", + "`flink-table-api-java`.`marketplace`.`order-placed-after-click`" + ); + + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + Arrays.stream(env.listTables()).forEach(System.out::println); + + customers.allCustomers(); + customers.allCustomerAddresses(); + orders.ordersOver50Dollars(); + orders.pricesWithTax(BigDecimal.valueOf(1.1)); + + orders.createFreeShippingTable(); + orders.streamOrdersOver50Dollars(); + + orders.createOrdersForPeriodTable(); + orders.streamOrdersForPeriod(Duration.ofMinutes(1)); + + clicks.createOrderPlacedAfterClickTable(); + clicks.streamOrderPlacedAfterClick(Duration.ofMinutes(5)); + } +} diff --git a/solutions/05-joins/src/main/java/marketplace/OrderService.java b/solutions/05-joins/src/main/java/marketplace/OrderService.java new file mode 100644 index 0000000..964d65e --- /dev/null +++ b/solutions/05-joins/src/main/java/marketplace/OrderService.java @@ -0,0 +1,117 @@ +package marketplace; + +import org.apache.flink.table.api.*; + +import java.math.BigDecimal; +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class OrderService { + private final TableEnvironment env; + private final String ordersTableName; + private final String freeShippingTableName; + private final String ordersForPeriodTableName; + + public OrderService( + TableEnvironment env, + String ordersTableName, + String freeShippingTableName, + String ordersForPeriodTableName + ) { + this.env = env; + this.ordersTableName = ordersTableName; + this.freeShippingTableName = freeShippingTableName; + this.ordersForPeriodTableName = ordersForPeriodTableName; + } + + public TableResult createFreeShippingTable() { + return env.executeSql( + "CREATE TABLE IF NOT EXISTS "+freeShippingTableName+" (\n" + + " `order_id` STRING NOT NULL,\n" + + " `details` ROW (\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` STRING NOT NULL,\n" + + " `price` DOUBLE NOT NULL \n" + + " ) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"); + } + + public TableResult createOrdersForPeriodTable() { + return env.executeSql( + "CREATE TABLE IF NOT EXISTS "+ordersForPeriodTableName+" (\n" + + " `customer_id` INT NOT NULL,\n" + + " `window_start` TIMESTAMP(3) NOT NULL,\n" + + " `window_end` TIMESTAMP(3) NOT NULL,\n" + + " `period_in_seconds` BIGINT NOT NULL,\n" + + " `product_ids` MULTISET NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");" + ); + } + + public TableResult ordersOver50Dollars() { + return env.from(ordersTableName) + .select($("*")) + .where($("price").isGreaterOrEqual(50)) + .execute(); + } + + public TableResult streamOrdersOver50Dollars() { + return env.from(ordersTableName) + .where($("price").isGreaterOrEqual(50)) + .select( + $("order_id"), + row( + $("customer_id"), + $("product_id"), + $("price") + ).as("details") + ) + .insertInto(freeShippingTableName) + .execute(); + } + + public TableResult pricesWithTax(BigDecimal taxAmount) { + return env.from(ordersTableName) + .select( + $("order_id"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .as("original_price"), + $("price") + .cast(DataTypes.DECIMAL(10, 2)) + .times(taxAmount) + .round(2) + .as("price_with_tax") + ).execute(); + } + + public TableResult streamOrdersForPeriod(Duration period) { + env.useCatalog("examples"); + env.useDatabase("marketplace"); + + return env.from(ordersTableName) + .window( + Tumble.over(lit(period.toSeconds()).seconds()) + .on($("$rowtime")) + .as("window") + ).groupBy( + $("customer_id"),$("window") + ).select( + $("customer_id"), + $("window").start(), + $("window").end(), + lit(period.toSeconds()).seconds().as("period_in_seconds"), + $("product_id").collect().as("product_ids") + ).insertInto( + ordersForPeriodTableName + ) + .execute(); + } +} diff --git a/solutions/05-joins/src/main/resources/cloud-template.properties b/solutions/05-joins/src/main/resources/cloud-template.properties new file mode 100644 index 0000000..528e4c2 --- /dev/null +++ b/solutions/05-joins/src/main/resources/cloud-template.properties @@ -0,0 +1,38 @@ +##################################################################### +# Confluent Cloud Connection Configuration # +# # +# Note: The plugin supports different ways of passing parameters: # +# - Programmatically # +# - Via global environment variables # +# - Via arguments in the main() method # +# - Via properties file # +# # +# For all cases, use the ConfluentSettings class to get started. # +# # +# The project is preconfigured with this properties file. # +##################################################################### + +# Cloud region +client.cloud= +client.region= + +# Access & compute resources +client.flink-api-key= +client.flink-api-secret= +client.organization-id= +client.environment-id= +client.compute-pool-id= + +# User or service account +client.principal-id= + +# Kafka (used by tests) +client.kafka.bootstrap.servers= +client.kafka.security.protocol=SASL_SSL +client.kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='' password=''; +client.kafka.sasl.mechanism=PLAIN + +# Schema Registry (used by tests) +client.registry.url= +client.registry.key= +client.registry.secret= \ No newline at end of file diff --git a/solutions/05-joins/src/main/resources/log4j2.properties b/solutions/05-joins/src/main/resources/log4j2.properties new file mode 100644 index 0000000..32c696e --- /dev/null +++ b/solutions/05-joins/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/solutions/05-joins/src/test/java/marketplace/ClickBuilder.java b/solutions/05-joins/src/test/java/marketplace/ClickBuilder.java new file mode 100644 index 0000000..0b47178 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/ClickBuilder.java @@ -0,0 +1,61 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +public class ClickBuilder { + private String clickId; + private int userId; + private String url; + private String userAgent; + private int viewTime; + private Instant timestamp; + + public ClickBuilder() { + Random rnd = new Random(); + + clickId = "Click"+rnd.nextInt(1000); + userId = rnd.nextInt(1000); + url = "http://some.url/"+rnd.nextInt(1000); + userAgent = "UserAgent"+rnd.nextInt(1000); + viewTime = rnd.nextInt(1000); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public ClickBuilder withClickId(String clickId) { + this.clickId = clickId; + return this; + } + + public ClickBuilder withUserId(int userId) { + this.userId = userId; + return this; + } + + public ClickBuilder withUrl(String url) { + this.url = url; + return this; + } + + public ClickBuilder withViewTime(int viewTime) { + this.viewTime = viewTime; + return this; + } + + public ClickBuilder withUserAgent(String user_agent) { + this.userAgent = user_agent; + return this; + } + + public ClickBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(clickId, userId, url, userAgent, viewTime, timestamp); + } +} diff --git a/solutions/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java b/solutions/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java new file mode 100644 index 0000000..fb691ae --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java @@ -0,0 +1,241 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("IntegrationTest") +class ClickServiceIntegrationTest extends FlinkIntegrationTest { + private final String clicksTableName = "`flink-table-api-java`.`marketplace`.`clicks-temp`"; + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderPlacedAfterClickTableName = "`flink-table-api-java`.`marketplace`.`order-placed-after-click-temp`"; + private final String orderPlacedAfterClickShortTableName = "order-placed-after-click-temp"; + + private final String clicksTableDefinition = + "CREATE TABLE IF NOT EXISTS " + clicksTableName + " (\n" + + " `click_id` VARCHAR(2147483647) NOT NULL,\n" + + " `user_id` INT NOT NULL,\n" + + " `url` VARCHAR(2147483647) NOT NULL,\n" + + " `user_agent` VARCHAR(2147483647) NOT NULL,\n" + + " `view_time` INT NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price", "event_time"); + private Integer indexOfOrderField(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + private final List clickTableFields = Arrays.asList("click_id", "user_id", "url", "user_agent", "view_time", "event_time"); + private Integer indexOfClickField(String fieldName) { + return clickTableFields.indexOf(fieldName); + } + + private ClickService clickService; + + @Override + public void setup() { + super.setup(); + clickService = new ClickService( + env, + clicksTableName, + ordersTableName, + orderPlacedAfterClickTableName + ); + } + + @Test + @Timeout(60) + public void createOrderPlacedAfterClickTable_shouldCreateTheTable() { + deleteTableOnExit(orderPlacedAfterClickTableName); + + TableResult result = clickService.createOrderPlacedAfterClickTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderPlacedAfterClickShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderPlacedAfterClickShortTableName), + "Could not find the table: "+orderPlacedAfterClickShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderPlacedAfterClickShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrderPlacedAfterClick_shouldJoinOrdersAndClicksAndEmitANewStream() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(clicksTableName); + deleteTable(ordersTableName); + deleteTable(orderPlacedAfterClickTableName); + + // Create the necessary tables. + createTemporaryTable(clicksTableName, clicksTableDefinition); + createTemporaryTable(ordersTableName, ordersTableDefinition); + clickService.createOrderPlacedAfterClickTable().await(); + deleteTableOnExit(orderPlacedAfterClickTableName); + + // Define some constants. + final Duration withinTimePeriod = Duration.ofMinutes(5); + final Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); + final Instant onTime = now.plusSeconds(1); + final Instant late = now.plus(withinTimePeriod).plusSeconds(1); + + // Define some customer Ids. + List customerIds = Arrays.asList(1, 2, 3, 4, 5); + + // Create some clicks. + List expectedClicks = customerIds.stream() + .map(customer -> + new ClickBuilder() + .withUserId(customer) + .withTimestamp(now) + .build() + ) + .toList(); + + // Mutable copy of the clicks. + List onTimeClicks = new ArrayList<>(expectedClicks); + + // Add a non-matching user Id. + onTimeClicks.add( + new ClickBuilder() + .withUserId(99) + .withTimestamp(now) + .build() + ); + + // Randomize the list. + Collections.shuffle(onTimeClicks); + + // Create some orders. + List expectedOrders = customerIds.stream() + .map(customer -> + new OrderBuilder() + .withCustomerId(customer) + .withTimestamp(onTime) + .build() + ) + .toList(); + + // Mutable copy of the orders + List onTimeOrders = new ArrayList<>(expectedOrders); + + // Add a non-matching customer Id. + onTimeOrders.add( + new OrderBuilder() + .withCustomerId(101) + .withTimestamp(onTime) + .build() + ); + + // Randomize the list. + Collections.shuffle(onTimeOrders); + + // Create a late click. + Row lateClick = new ClickBuilder() + .withUserId(1) + .withTimestamp(late) + .build(); + + // Create a late order. + Row lateOrder = new OrderBuilder() + .withCustomerId(5) + .withTimestamp(late) + .build(); + + // Push data into the destination tables. + env.fromValues(onTimeClicks).insertInto(clicksTableName).execute(); + env.fromValues(onTimeOrders).insertInto(ordersTableName).execute(); + + // We push the late data separately, to ensure it actually comes after the earlier data. + env.fromValues(lateClick).insertInto(clicksTableName).execute(); + env.fromValues(lateOrder).insertInto(ordersTableName).execute(); + + // Execute the query we are testing. + cancelOnExit(clickService.streamOrderPlacedAfterClick(withinTimePeriod)); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderPlacedAfterClickTableName) + .select($("*")) + .execute() + ); + + Set actual = new HashSet<>( + fetchRows(queryResult) + .limit(customerIds.size()) + .toList() + ); + + // Build the expected results. + Set expected = new HashSet<> ( + customerIds.stream().map(customer -> { + Row clickRow = expectedClicks.stream() + .filter(click -> click.getFieldAs(indexOfClickField("user_id")).equals(customer)) + .findFirst() + .orElseThrow(); + + Row orderRow = expectedOrders.stream() + .filter(click -> click.getFieldAs(indexOfOrderField("customer_id")).equals(customer)) + .findFirst() + .orElseThrow(); + + return Row.of( + customer, + clickRow.getField(indexOfClickField("url")), + clickRow.getField(indexOfClickField("event_time")), + orderRow.getField(indexOfOrderField("product_id")), + orderRow.getField(indexOfOrderField("event_time")) + ); + }).toList() + ); + + // Assert on the results. + assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/solutions/05-joins/src/test/java/marketplace/ClickServiceTest.java b/solutions/05-joins/src/test/java/marketplace/ClickServiceTest.java new file mode 100644 index 0000000..f7ddcfb --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/ClickServiceTest.java @@ -0,0 +1,123 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +class ClickServiceTest { + private ClickService service; + private TableEnvironment mockEnv; + private Table mockClicksTable; + private Table mockOrdersTable; + private Table mockJoinedTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockClicksTable = mock(Table.class); + mockOrdersTable = mock(Table.class); + mockJoinedTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + service = new ClickService( + mockEnv, + "clickTable", + "orderTable", + "orderPlacedAfterClickTable" + ); + + when(mockEnv.from("clickTable")).thenReturn(mockClicksTable); + when(mockClicksTable.select(any(Expression[].class))).thenReturn(mockClicksTable); + when(mockEnv.from("orderTable")).thenReturn(mockOrdersTable); + when(mockOrdersTable.select(any(Expression[].class))).thenReturn(mockOrdersTable); + when(mockClicksTable.join(any())).thenReturn(mockJoinedTable); + when(mockOrdersTable.join(any())).thenReturn(mockJoinedTable); + when(mockJoinedTable.where(any())).thenReturn(mockJoinedTable); + when(mockJoinedTable.select(any(Expression[].class))).thenReturn(mockJoinedTable); + when(mockJoinedTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void createOrderPlacedAfterClickTable_shouldSendTheExpectedSQL() { + TableResult result = service.createOrderPlacedAfterClickTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("orderPlacedAfterClickTable")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("clicked_url")); + assertTrue(sql.contains("time_of_click")); + assertTrue(sql.contains("purchased_product")); + assertTrue(sql.contains("time_of_order")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrderPlacedAfterClick_shouldStreamTheExpectedRecordsToTheTable() { + Duration withinTimePeriod = Duration.ofMinutes(5); + TableResult result = service.streamOrderPlacedAfterClick(withinTimePeriod); + + verify(mockEnv).from("clickTable"); + verify(mockEnv).from("orderTable"); + + ArgumentCaptor whereCaptor = ArgumentCaptor.forClass(Expression.class); + verify(mockJoinedTable).where(whereCaptor.capture()); + String whereExpression = whereCaptor.getValue().asSummaryString(); + + assertTrue(whereExpression.contains("and")); + assertTrue( + whereExpression.contains("equals(user_id, customer_id") + || whereExpression.contains("equals(customer_id, user_id)") + ); + assertTrue(whereExpression.contains(Long.toString(withinTimePeriod.toSeconds()))); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockJoinedTable).select(selectCaptor.capture()); + List selectExpressions = Arrays.stream(selectCaptor.getValue()) + .map(Expression::asSummaryString) + .toList(); + + assertTrue(selectExpressions.get(0).contains("customer_id")); + assertTrue(selectExpressions.get(1).contains("clicked_url")); + assertTrue(selectExpressions.get(2).contains("time_of_click")); + assertTrue(selectExpressions.get(3).contains("purchased_product")); + assertTrue(selectExpressions.get(4).contains("time_of_order")); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockJoinedTable).insertInto(insertCaptor.capture()); + + assertEquals( + "orderPlacedAfterClickTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } + +} \ No newline at end of file diff --git a/solutions/05-joins/src/test/java/marketplace/CustomerBuilder.java b/solutions/05-joins/src/test/java/marketplace/CustomerBuilder.java new file mode 100644 index 0000000..5bbf318 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/CustomerBuilder.java @@ -0,0 +1,59 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.util.Random; + +class CustomerBuilder { + private int customerId; + private String name; + private String address; + private String postCode; + private String city; + private String email; + + private final Random rnd = new Random(System.currentTimeMillis()); + + public CustomerBuilder() { + customerId = rnd.nextInt(1000); + name = "Name" + rnd.nextInt(1000); + address = "Address" + rnd.nextInt(1000); + postCode = "PostCode" + rnd.nextInt(1000); + city = "City" + rnd.nextInt(1000); + email = "Email" + rnd.nextInt(1000); + } + + public CustomerBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public CustomerBuilder withName(String name) { + this.name = name; + return this; + } + + public CustomerBuilder withAddress(String address) { + this.address = address; + return this; + } + + public CustomerBuilder withPostCode(String postCode) { + this.postCode = postCode; + return this; + } + + public CustomerBuilder withCity(String city) { + this.city = city; + return this; + } + + public CustomerBuilder withEmail(String email) { + this.email = email; + return this; + } + + public Row build() { + return Row.of(customerId, name, address, postCode, city, email); + } +} diff --git a/solutions/05-joins/src/test/java/marketplace/CustomerServiceIntegrationTest.java b/solutions/05-joins/src/test/java/marketplace/CustomerServiceIntegrationTest.java new file mode 100644 index 0000000..34a4d28 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/CustomerServiceIntegrationTest.java @@ -0,0 +1,114 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class CustomerServiceIntegrationTest extends FlinkIntegrationTest { + private final String customersTableName = "`flink-table-api-java`.`marketplace`.`customers-temp`"; + + private final String customersTableDefinition = + "CREATE TABLE " + customersTableName + " (\n" + + " `customer_id` INT NOT NULL,\n" + + " `name` VARCHAR(2147483647) NOT NULL,\n" + + " `address` VARCHAR(2147483647) NOT NULL,\n" + + " `postcode` VARCHAR(2147483647) NOT NULL,\n" + + " `city` VARCHAR(2147483647) NOT NULL,\n" + + " `email` VARCHAR(2147483647) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private CustomerService customerService; + + @Override + public void setup() { + customerService = new CustomerService( + env, + customersTableName + ); + } + + @Test + @Timeout(90) + public void allCustomers_shouldReturnTheDetailsOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomers()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(customers), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id","name", "address", "postcode", "city", "email" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } + + @Test + @Timeout(90) + public void allCustomerAddresses_shouldReturnTheAddressesOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomerAddresses()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(customers.size(), actual.size()); + + List expected = customers.stream() + .map(row -> Row.project(row, new int[] {0, 2, 3, 4})) + .toList(); + + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id", "address", "postcode", "city" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } +} \ No newline at end of file diff --git a/solutions/05-joins/src/test/java/marketplace/CustomerServiceTest.java b/solutions/05-joins/src/test/java/marketplace/CustomerServiceTest.java new file mode 100644 index 0000000..081b1d0 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/CustomerServiceTest.java @@ -0,0 +1,67 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@Tag("UnitTest") +public class CustomerServiceTest { + + private CustomerService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockResult = mock(TableResult.class); + service = new CustomerService(mockEnv, "customerTable"); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.execute()).thenReturn(mockResult); + } + + @Test + public void allCustomers_shouldSelectAllFields() { + TableResult result = service.allCustomers(); + + verify(mockEnv).from("customerTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + assertEquals(mockResult, result); + } + + @Test + public void allCustomerAddresses_shouldSelectOnlyTheRelevantFields() { + TableResult result = service.allCustomerAddresses(); + + verify(mockEnv).from("customerTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] {"customer_id", "address", "postcode", "city"}, selectArgs.toArray()); + + assertEquals(mockResult, result); + } +} diff --git a/solutions/05-joins/src/test/java/marketplace/FlinkIntegrationTest.java b/solutions/05-joins/src/test/java/marketplace/FlinkIntegrationTest.java new file mode 100644 index 0000000..f919c37 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/FlinkIntegrationTest.java @@ -0,0 +1,189 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.common.KafkaFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class FlinkIntegrationTest { + private List jobsToCancel; + private List topicsToDelete; + private Thread shutdownHook; + private boolean isShuttingDown; + + protected TableEnvironment env; + protected AdminClient adminClient; + protected SchemaRegistryClient registryClient; + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected void setup() {}; + protected void teardown() {}; + + @BeforeEach + public void mainSetup() throws Exception { + jobsToCancel = new ArrayList<>(); + topicsToDelete = new ArrayList<>(); + + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + env = TableEnvironment.create(settings); + + Properties properties = new Properties(); + settings.getConfiguration().toMap().forEach((k,v) -> + properties.put(k.replace("client.kafka.", ""), v) + ); + adminClient = AdminClient.create(properties); + + Map schemaConfig = new HashMap<>(); + + schemaConfig.put("basic.auth.credentials.source", "USER_INFO"); + schemaConfig.put( + "basic.auth.user.info", + properties.get("client.registry.key") + ":" + properties.get("client.registry.secret")); + + registryClient = new CachedSchemaRegistryClient( + properties.getProperty("client.registry.url"), + 100, + schemaConfig + ); + + isShuttingDown = false; + shutdownHook = new Thread(() -> { + logger.info("Shutdown Detected. Cleaning up resources."); + isShuttingDown = true; + mainTeardown(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + setup(); + } + + @AfterEach + public void mainTeardown() { + teardown(); + + jobsToCancel.forEach(result -> + result.getJobClient() + .orElseThrow() + .cancel() + .join() + ); + + topicsToDelete.forEach(topic -> + deleteTopic(topic) + ); + + if(!isShuttingDown) { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } + } + + protected TableResult retry(Supplier supplier) { + return retry(3, supplier); + } + + protected TableResult retry(int tries, Supplier supplier) { + try { + return supplier.get(); + } catch (Exception e) { + logger.error("Failed on retryable command.", e); + + if(tries > 0) { + logger.info("Retrying"); + return retry(tries - 1, supplier); + } else { + logger.info("Maximum number of tries exceeded. Failing..."); + throw e; + } + } + } + + protected TableResult cancelOnExit(TableResult tableResult) { + jobsToCancel.add(tableResult); + return tableResult; + } + + protected Stream fetchRows(TableResult result) { + Iterable iterable = result::collect; + return StreamSupport.stream(iterable.spliterator(), false); + } + + protected String getShortTableName(String tableName) { + String[] tablePath = tableName.split("\\."); + return tablePath[tablePath.length - 1].replace("`",""); + } + + protected void deleteTopic(String topicName) { + try { + String schemaName = topicName + "-value"; + logger.info("Deleting Schema: "+schemaName); + if(registryClient.getAllSubjects().contains(schemaName)) { + registryClient.deleteSubject(schemaName, false); + registryClient.deleteSubject(schemaName, true); + } + logger.info("Deleted Schema: "+schemaName); + } catch (Exception e) { + logger.error("Error Deleting Schema", e); + } + + try { + if(adminClient.listTopics().names().get().contains(topicName)) { + logger.info("Deleting Topic: " + topicName); + KafkaFuture result = adminClient.deleteTopics(List.of(topicName)).all(); + + while(!result.isDone()) { + logger.info("Waiting for topic to be deleted: " + topicName); + Thread.sleep(1000); + } + + logger.info("Topic Deleted: " + topicName); + } + } catch (Exception e) { + logger.error("Error Deleting Topic", e); + } + } + + protected void deleteTable(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopic(topicName); + } + + protected void deleteTopicOnExit(String topicName) { + topicsToDelete.add(topicName); + } + + protected void deleteTableOnExit(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopicOnExit(topicName); + } + + protected void createTemporaryTable(String fullyQualifiedTableName, String tableDefinition) { + String topicName = getShortTableName(fullyQualifiedTableName); + + logger.info("Creating temporary table: " + fullyQualifiedTableName); + + try { + env.executeSql(tableDefinition).await(); + deleteTopicOnExit(topicName); + + logger.info("Created temporary table: " + fullyQualifiedTableName); + } catch (Exception e) { + logger.error("Unable to create temporary table: " + fullyQualifiedTableName, e); + } + } +} diff --git a/solutions/05-joins/src/test/java/marketplace/OrderBuilder.java b/solutions/05-joins/src/test/java/marketplace/OrderBuilder.java new file mode 100644 index 0000000..1b5e2e6 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/OrderBuilder.java @@ -0,0 +1,52 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +class OrderBuilder { + private String orderId, productId; + private Integer customerId; + private Double price; + private Instant timestamp; + + public OrderBuilder() { + Random rnd = new Random(); + orderId = "Order" + rnd.nextInt(1000); + customerId = rnd.nextInt(1000); + productId = "Product" + rnd.nextInt(1000); + price = rnd.nextDouble(100); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public OrderBuilder withOrderId(String orderId) { + this.orderId = orderId; + return this; + } + + public OrderBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public OrderBuilder withProductId(String productId) { + this.productId = productId; + return this; + } + + public OrderBuilder withPrice(Double price) { + this.price = price; + return this; + } + + public OrderBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(orderId, customerId, productId, price, timestamp); + } +} diff --git a/solutions/05-joins/src/test/java/marketplace/OrderServiceIntegrationTest.java b/solutions/05-joins/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..e8c4467 --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,365 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderQualifiedForFreeShippingTableName = "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping-temp`"; + private final String orderQualifiedForFreeShippingShortTableName = "order-qualified-for-free-shipping-temp"; + private final String customerOrdersForPeriodTableName = "`flink-table-api-java`.`marketplace`.`customer-orders-collected-for-period-temp`"; + private final String customerOrdersForPeriodShortTableName = "customer-orders-collected-for-period-temp"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + private Row toQualifiedForFreeShippingRow(Row row) { + return Row.of( + row.getFieldAs(indexOf("order_id")), + Row.of( + row.getFieldAs(indexOf("customer_id")), + row.getFieldAs(indexOf("product_id")), + row.getFieldAs(indexOf("price")) + ) + ); + } + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName, + orderQualifiedForFreeShippingTableName, + customerOrdersForPeriodTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(60) + public void createFreeShippingTable_shouldCreateTheTable() { + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + TableResult result = orderService.createFreeShippingTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderQualifiedForFreeShippingShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderQualifiedForFreeShippingShortTableName), + "Could not find the table: "+orderQualifiedForFreeShippingShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderQualifiedForFreeShippingShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrdersOver50Dollars_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(orderQualifiedForFreeShippingTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the destination table. + orderService.createFreeShippingTable().await(); + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + final int detailsPosition = 1; + + // Create a list of orders with specific prices. + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Initiate the stream. + cancelOnExit(orderService.streamOrdersOver50Dollars()); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderQualifiedForFreeShippingTableName) + .select($("*")) + .execute() + ); + + // Obtain the actual results. + List actual = fetchRows(queryResult) + .limit(Arrays.stream(prices).filter(p -> p >= 50).count()) + .toList(); + + // Build the expected results + List expected = orders.stream() + .filter(row -> row.getFieldAs(indexOf("price")) >= 50) + .map(this::toQualifiedForFreeShippingRow) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + assertEquals( + new HashSet<>(Arrays.asList("order_id", "details")), + actual.getFirst().getFieldNames(true) + ); + + assertEquals( + new HashSet<>(Arrays.asList("customer_id", "product_id", "price")), + actual.getFirst().getFieldAs(detailsPosition).getFieldNames(true) + ); + } + + @Test + @Timeout(60) + public void createOrdersForPeriodTable_shouldCreateTheTable() { + deleteTableOnExit(customerOrdersForPeriodTableName); + + TableResult result = orderService.createOrdersForPeriodTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+customerOrdersForPeriodShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(customerOrdersForPeriodShortTableName), + "Could not find the table: "+customerOrdersForPeriodShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE "+customerOrdersForPeriodTableName + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamCustomerPurchasesDuringPeriod_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(customerOrdersForPeriodTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the output table. + orderService.createOrdersForPeriodTable().await(); + deleteTableOnExit(customerOrdersForPeriodTableName); + + final Duration windowSize = Duration.ofSeconds(10); + + // Create a set of products for each customer. + Map> customerProducts = new HashMap<>(); + customerProducts.put(1, Arrays.asList("Product1", "Product2")); + customerProducts.put(2, Arrays.asList("Product3", "Product4")); + customerProducts.put(3, Arrays.asList("Product5")); + customerProducts.put(4, Arrays.asList("Product6", "Product7")); + customerProducts.put(5, Arrays.asList("Product8", "Product9","Product9")); // Product9 is duplicated. + + // Create an order for each product. + List orders = customerProducts.entrySet().stream() + .flatMap(entry -> + entry.getValue().stream().map(v -> + new OrderBuilder() + .withCustomerId(entry.getKey()) + .withProductId(v) + .withTimestamp(Instant.now().truncatedTo(ChronoUnit.MILLIS).minus(windowSize)) + .build() + ) + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Flink only evaluates windows when a new event arrives. + // Therefore, in order to trigger the window closing, we need to issue + // another event after the window period. Because we are specifying the + // other events to occur 10 seconds early, just issuing an event now + // will do the trick. + env.fromValues(new OrderBuilder().build()).insertInto(ordersTableName).execute(); + + // Execute the method being tested. + cancelOnExit(orderService.streamOrdersForPeriod(windowSize)); + + // Fetch the results. + TableResult results = retry(() -> + env.from(customerOrdersForPeriodTableName) + .select($("*")) + .execute() + ); + + List actual = fetchRows(results) + .limit(customerProducts.size()).toList(); + + // Assert on the results. + actual.forEach(row -> { + Integer customerId = row.getFieldAs("customer_id"); + + Map actualProducts = row.getFieldAs("product_ids"); + Map expectedProducts = customerProducts.get(customerId).stream() + .collect(Collectors.groupingBy(product -> product, Collectors.summingInt(x -> 1))); + + assertEquals(expectedProducts, actualProducts); + assertEquals( + row.getFieldAs("window_start"), + row.getFieldAs("window_end").minus(windowSize) + ); + assertEquals(windowSize.toSeconds(), row.getFieldAs("period_in_seconds")); + }); + } +} \ No newline at end of file diff --git a/solutions/05-joins/src/test/java/marketplace/OrderServiceTest.java b/solutions/05-joins/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..01429cd --- /dev/null +++ b/solutions/05-joins/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,215 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable", + "freeShippingTable", + "ordersForPeriodTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } + + @Test + public void createFreeShippingTable_shouldSendTheExpectedSQL() { + TableResult result = service.createFreeShippingTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("freeShippingTable")); + assertTrue(sql.contains("details")); + assertTrue(sql.toLowerCase().contains("row")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("product_id")); + assertTrue(sql.contains("price")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrdersOver50Dollars_shouldStreamTheExpectedRecordsToTheTable() { + TableResult result = service.streamOrdersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(2, expressions.length); + assertEquals("order_id", expressions[0].asSummaryString()); + assertEquals("as(row(customer_id, product_id, price), 'details')", expressions[1].asSummaryString()); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "freeShippingTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } + + @Test + public void createOrdersForPeriodTable_shouldSendTheExpectedSQL() { + TableResult result = service.createOrdersForPeriodTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("ordersForPeriodTable")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("window_start")); + assertTrue(sql.contains("window_end")); + assertTrue(sql.contains("period_in_seconds")); + assertTrue(sql.contains("product_ids")); + + assertEquals(mockResult, result); + } + + @Test + public void streamCustomerPurchasesDuringPeriod_shouldStreamTheExpectedRecordsToTheTable() { + Duration windowSize = Duration.ofSeconds(10); + TableResult result = service.streamOrdersForPeriod(windowSize); + + verify(mockEnv).useCatalog("examples"); + verify(mockEnv).useDatabase("marketplace"); + verify(mockEnv).from("orderTable"); + + ArgumentCaptor windowCaptor = ArgumentCaptor.forClass(GroupWindow.class); + verify(mockTable).window(windowCaptor.capture()); + + TumbleWithSizeOnTimeWithAlias window = (TumbleWithSizeOnTimeWithAlias) windowCaptor.getValue(); + String windowAlias = window.getAlias().asSummaryString(); + String timeField = window.getTimeField().asSummaryString(); + long size = Long.parseLong(window.getSize().asSummaryString()); + + assertEquals("$rowtime", timeField); + assertEquals(10000l, size); + + ArgumentCaptor groupByCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockGroupWindowedTable).groupBy(groupByCaptor.capture()); + Expression[] groupByExpressions = groupByCaptor.getValue(); + + assertEquals(2, groupByExpressions.length); + assertEquals("customer_id", groupByExpressions[0].asSummaryString()); + assertEquals(windowAlias, groupByExpressions[1].asSummaryString()); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockWindowGroupedTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(5, expressions.length); + assertEquals("customer_id", expressions[0].asSummaryString()); + assertEquals("start("+windowAlias+")", expressions[1].asSummaryString()); + assertEquals("end("+windowAlias+")", expressions[2].asSummaryString()); + assertEquals("as("+lit(windowSize.getSeconds()).seconds().asSummaryString()+", 'period_in_seconds')" , expressions[3].asSummaryString()); + assertEquals("as(collect(product_id), 'product_ids')", expressions[4].asSummaryString()); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "ordersForPeriodTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } +} diff --git a/staging/01-connecting-to-confluent-cloud/pom.xml b/staging/01-connecting-to-confluent-cloud/pom.xml new file mode 100644 index 0000000..2654828 --- /dev/null +++ b/staging/01-connecting-to-confluent-cloud/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.129.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + 5.11.0 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-flink-table-api-java-plugin + ${confluent-plugin.version} + + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.0 + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + + + + 5 + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/staging/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java b/staging/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java new file mode 100644 index 0000000..a7151b4 --- /dev/null +++ b/staging/01-connecting-to-confluent-cloud/src/main/java/marketplace/Marketplace.java @@ -0,0 +1,17 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; + +import java.io.File; +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; + +public class Marketplace { + + public static void main(String[] args) throws Exception { + // TODO + } +} diff --git a/staging/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties b/staging/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties new file mode 100644 index 0000000..8047170 --- /dev/null +++ b/staging/01-connecting-to-confluent-cloud/src/main/resources/cloud-template.properties @@ -0,0 +1,38 @@ +##################################################################### +# Confluent Cloud Connection Configuration # +# # +# Note: The plugin supports different ways of passing parameters: # +# - Programmatically # +# - Via global environment variables # +# - Via arguments in the main() method # +# - Via properties file # +# # +# For all cases, use the ConfluentSettings class to get started. # +# # +# The project is preconfigured with this properties file. # +##################################################################### + +# Cloud region +client.cloud= +client.region= + +# Access & compute resources +client.flink-api-key= +client.flink-api-secret= +client.organization-id= +client.environment-id= +client.compute-pool-id= + +# User or service account +client.principal-id= + +# Kafka (used by tests) +client.kafka.bootstrap.servers= +client.kafka.security.protocol=SASL_SSL +client.kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='' password=''; +client.kafka.sasl.mechanism=PLAIN + +# Schema Registry (used by tests) +client.registry.url= +client.registry.key= +client.registry.secret= \ No newline at end of file diff --git a/staging/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties b/staging/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties new file mode 100644 index 0000000..32c696e --- /dev/null +++ b/staging/01-connecting-to-confluent-cloud/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/staging/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java b/staging/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java new file mode 100644 index 0000000..eb8e560 --- /dev/null +++ b/staging/02-querying-flink-tables/src/main/java/marketplace/CustomerService.java @@ -0,0 +1,29 @@ +package marketplace; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import static org.apache.flink.table.api.Expressions.$; + +public class CustomerService { + private final TableEnvironment env; + private final String customersTableName; + + public CustomerService( + TableEnvironment env, + String customersTableName + ) { + this.env = env; + this.customersTableName = customersTableName; + } + + public TableResult allCustomers() { + // TODO + return null; + } + + public TableResult allCustomerAddresses() { + // TODO + return null; + } +} diff --git a/staging/02-querying-flink-tables/src/main/java/marketplace/OrderService.java b/staging/02-querying-flink-tables/src/main/java/marketplace/OrderService.java new file mode 100644 index 0000000..c67111d --- /dev/null +++ b/staging/02-querying-flink-tables/src/main/java/marketplace/OrderService.java @@ -0,0 +1,31 @@ +package marketplace; + +import org.apache.flink.table.api.*; + +import java.math.BigDecimal; +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class OrderService { + private final TableEnvironment env; + private final String ordersTableName; + + public OrderService( + TableEnvironment env, + String ordersTableName + ) { + this.env = env; + this.ordersTableName = ordersTableName; + } + + public TableResult ordersOver50Dollars() { + // TODO + return null; + } + + public TableResult pricesWithTax(BigDecimal taxAmount) { + // TODO + return null; + } +} diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java b/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java new file mode 100644 index 0000000..5bbf318 --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerBuilder.java @@ -0,0 +1,59 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.util.Random; + +class CustomerBuilder { + private int customerId; + private String name; + private String address; + private String postCode; + private String city; + private String email; + + private final Random rnd = new Random(System.currentTimeMillis()); + + public CustomerBuilder() { + customerId = rnd.nextInt(1000); + name = "Name" + rnd.nextInt(1000); + address = "Address" + rnd.nextInt(1000); + postCode = "PostCode" + rnd.nextInt(1000); + city = "City" + rnd.nextInt(1000); + email = "Email" + rnd.nextInt(1000); + } + + public CustomerBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public CustomerBuilder withName(String name) { + this.name = name; + return this; + } + + public CustomerBuilder withAddress(String address) { + this.address = address; + return this; + } + + public CustomerBuilder withPostCode(String postCode) { + this.postCode = postCode; + return this; + } + + public CustomerBuilder withCity(String city) { + this.city = city; + return this; + } + + public CustomerBuilder withEmail(String email) { + this.email = email; + return this; + } + + public Row build() { + return Row.of(customerId, name, address, postCode, city, email); + } +} diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java b/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java new file mode 100644 index 0000000..34a4d28 --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceIntegrationTest.java @@ -0,0 +1,114 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class CustomerServiceIntegrationTest extends FlinkIntegrationTest { + private final String customersTableName = "`flink-table-api-java`.`marketplace`.`customers-temp`"; + + private final String customersTableDefinition = + "CREATE TABLE " + customersTableName + " (\n" + + " `customer_id` INT NOT NULL,\n" + + " `name` VARCHAR(2147483647) NOT NULL,\n" + + " `address` VARCHAR(2147483647) NOT NULL,\n" + + " `postcode` VARCHAR(2147483647) NOT NULL,\n" + + " `city` VARCHAR(2147483647) NOT NULL,\n" + + " `email` VARCHAR(2147483647) NOT NULL\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private CustomerService customerService; + + @Override + public void setup() { + customerService = new CustomerService( + env, + customersTableName + ); + } + + @Test + @Timeout(90) + public void allCustomers_shouldReturnTheDetailsOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomers()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(customers), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id","name", "address", "postcode", "city", "email" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } + + @Test + @Timeout(90) + public void allCustomerAddresses_shouldReturnTheAddressesOfAllCustomers() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(customersTableName); + + // Create a temporary customers table. + createTemporaryTable(customersTableName, customersTableDefinition); + + // Generate some customers. + List customers = Stream.generate(() -> new CustomerBuilder().build()) + .limit(5) + .toList(); + + // Push the customers into the temporary table. + env.fromValues(customers).insertInto(customersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> customerService.allCustomerAddresses()); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(customers.size()) + .toList(); + + // Assert on the results. + assertEquals(customers.size(), actual.size()); + + List expected = customers.stream() + .map(row -> Row.project(row, new int[] {0, 2, 3, 4})) + .toList(); + + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "customer_id", "address", "postcode", "city" + )); + assertEquals(expectedFields, actual.getFirst().getFieldNames(true)); + } +} \ No newline at end of file diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java b/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java new file mode 100644 index 0000000..081b1d0 --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/CustomerServiceTest.java @@ -0,0 +1,67 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@Tag("UnitTest") +public class CustomerServiceTest { + + private CustomerService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockResult = mock(TableResult.class); + service = new CustomerService(mockEnv, "customerTable"); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.execute()).thenReturn(mockResult); + } + + @Test + public void allCustomers_shouldSelectAllFields() { + TableResult result = service.allCustomers(); + + verify(mockEnv).from("customerTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + assertEquals(mockResult, result); + } + + @Test + public void allCustomerAddresses_shouldSelectOnlyTheRelevantFields() { + TableResult result = service.allCustomerAddresses(); + + verify(mockEnv).from("customerTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] {"customer_id", "address", "postcode", "city"}, selectArgs.toArray()); + + assertEquals(mockResult, result); + } +} diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java b/staging/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java new file mode 100644 index 0000000..f919c37 --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/FlinkIntegrationTest.java @@ -0,0 +1,189 @@ +package marketplace; + +import io.confluent.flink.plugin.ConfluentSettings; +import org.apache.flink.table.api.EnvironmentSettings; +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.common.KafkaFuture; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class FlinkIntegrationTest { + private List jobsToCancel; + private List topicsToDelete; + private Thread shutdownHook; + private boolean isShuttingDown; + + protected TableEnvironment env; + protected AdminClient adminClient; + protected SchemaRegistryClient registryClient; + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected void setup() {}; + protected void teardown() {}; + + @BeforeEach + public void mainSetup() throws Exception { + jobsToCancel = new ArrayList<>(); + topicsToDelete = new ArrayList<>(); + + EnvironmentSettings settings = ConfluentSettings.fromResource("/cloud.properties"); + env = TableEnvironment.create(settings); + + Properties properties = new Properties(); + settings.getConfiguration().toMap().forEach((k,v) -> + properties.put(k.replace("client.kafka.", ""), v) + ); + adminClient = AdminClient.create(properties); + + Map schemaConfig = new HashMap<>(); + + schemaConfig.put("basic.auth.credentials.source", "USER_INFO"); + schemaConfig.put( + "basic.auth.user.info", + properties.get("client.registry.key") + ":" + properties.get("client.registry.secret")); + + registryClient = new CachedSchemaRegistryClient( + properties.getProperty("client.registry.url"), + 100, + schemaConfig + ); + + isShuttingDown = false; + shutdownHook = new Thread(() -> { + logger.info("Shutdown Detected. Cleaning up resources."); + isShuttingDown = true; + mainTeardown(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + setup(); + } + + @AfterEach + public void mainTeardown() { + teardown(); + + jobsToCancel.forEach(result -> + result.getJobClient() + .orElseThrow() + .cancel() + .join() + ); + + topicsToDelete.forEach(topic -> + deleteTopic(topic) + ); + + if(!isShuttingDown) { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } + } + + protected TableResult retry(Supplier supplier) { + return retry(3, supplier); + } + + protected TableResult retry(int tries, Supplier supplier) { + try { + return supplier.get(); + } catch (Exception e) { + logger.error("Failed on retryable command.", e); + + if(tries > 0) { + logger.info("Retrying"); + return retry(tries - 1, supplier); + } else { + logger.info("Maximum number of tries exceeded. Failing..."); + throw e; + } + } + } + + protected TableResult cancelOnExit(TableResult tableResult) { + jobsToCancel.add(tableResult); + return tableResult; + } + + protected Stream fetchRows(TableResult result) { + Iterable iterable = result::collect; + return StreamSupport.stream(iterable.spliterator(), false); + } + + protected String getShortTableName(String tableName) { + String[] tablePath = tableName.split("\\."); + return tablePath[tablePath.length - 1].replace("`",""); + } + + protected void deleteTopic(String topicName) { + try { + String schemaName = topicName + "-value"; + logger.info("Deleting Schema: "+schemaName); + if(registryClient.getAllSubjects().contains(schemaName)) { + registryClient.deleteSubject(schemaName, false); + registryClient.deleteSubject(schemaName, true); + } + logger.info("Deleted Schema: "+schemaName); + } catch (Exception e) { + logger.error("Error Deleting Schema", e); + } + + try { + if(adminClient.listTopics().names().get().contains(topicName)) { + logger.info("Deleting Topic: " + topicName); + KafkaFuture result = adminClient.deleteTopics(List.of(topicName)).all(); + + while(!result.isDone()) { + logger.info("Waiting for topic to be deleted: " + topicName); + Thread.sleep(1000); + } + + logger.info("Topic Deleted: " + topicName); + } + } catch (Exception e) { + logger.error("Error Deleting Topic", e); + } + } + + protected void deleteTable(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopic(topicName); + } + + protected void deleteTopicOnExit(String topicName) { + topicsToDelete.add(topicName); + } + + protected void deleteTableOnExit(String tableName) { + String topicName = getShortTableName(tableName); + deleteTopicOnExit(topicName); + } + + protected void createTemporaryTable(String fullyQualifiedTableName, String tableDefinition) { + String topicName = getShortTableName(fullyQualifiedTableName); + + logger.info("Creating temporary table: " + fullyQualifiedTableName); + + try { + env.executeSql(tableDefinition).await(); + deleteTopicOnExit(topicName); + + logger.info("Created temporary table: " + fullyQualifiedTableName); + } catch (Exception e) { + logger.error("Unable to create temporary table: " + fullyQualifiedTableName, e); + } + } +} diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java b/staging/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java new file mode 100644 index 0000000..1b5e2e6 --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/OrderBuilder.java @@ -0,0 +1,52 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +class OrderBuilder { + private String orderId, productId; + private Integer customerId; + private Double price; + private Instant timestamp; + + public OrderBuilder() { + Random rnd = new Random(); + orderId = "Order" + rnd.nextInt(1000); + customerId = rnd.nextInt(1000); + productId = "Product" + rnd.nextInt(1000); + price = rnd.nextDouble(100); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public OrderBuilder withOrderId(String orderId) { + this.orderId = orderId; + return this; + } + + public OrderBuilder withCustomerId(int customerId) { + this.customerId = customerId; + return this; + } + + public OrderBuilder withProductId(String productId) { + this.productId = productId; + return this; + } + + public OrderBuilder withPrice(Double price) { + this.price = price; + return this; + } + + public OrderBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(orderId, customerId, productId, price, timestamp); + } +} diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java b/staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..f9d619c --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,147 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } +} \ No newline at end of file diff --git a/staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java b/staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..6788739 --- /dev/null +++ b/staging/02-querying-flink-tables/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,94 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } +} diff --git a/staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java b/staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..6e9e87f --- /dev/null +++ b/staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,254 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderQualifiedForFreeShippingTableName = "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping-temp`"; + private final String orderQualifiedForFreeShippingShortTableName = "order-qualified-for-free-shipping-temp"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + private Row toQualifiedForFreeShippingRow(Row row) { + return Row.of( + row.getFieldAs(indexOf("order_id")), + Row.of( + row.getFieldAs(indexOf("customer_id")), + row.getFieldAs(indexOf("product_id")), + row.getFieldAs(indexOf("price")) + ) + ); + } + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName, + orderQualifiedForFreeShippingTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(60) + public void createFreeShippingTable_shouldCreateTheTable() { + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + TableResult result = orderService.createFreeShippingTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderQualifiedForFreeShippingShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderQualifiedForFreeShippingShortTableName), + "Could not find the table: "+orderQualifiedForFreeShippingShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderQualifiedForFreeShippingShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrdersOver50Dollars_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(orderQualifiedForFreeShippingTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the destination table. + orderService.createFreeShippingTable().await(); + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + final int detailsPosition = 1; + + // Create a list of orders with specific prices. + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Initiate the stream. + cancelOnExit(orderService.streamOrdersOver50Dollars()); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderQualifiedForFreeShippingTableName) + .select($("*")) + .execute() + ); + + // Obtain the actual results. + List actual = fetchRows(queryResult) + .limit(Arrays.stream(prices).filter(p -> p >= 50).count()) + .toList(); + + // Build the expected results + List expected = orders.stream() + .filter(row -> row.getFieldAs(indexOf("price")) >= 50) + .map(this::toQualifiedForFreeShippingRow) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + assertEquals( + new HashSet<>(Arrays.asList("order_id", "details")), + actual.getFirst().getFieldNames(true) + ); + + assertEquals( + new HashSet<>(Arrays.asList("customer_id", "product_id", "price")), + actual.getFirst().getFieldAs(detailsPosition).getFieldNames(true) + ); + } +} \ No newline at end of file diff --git a/staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java b/staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..7ef6e72 --- /dev/null +++ b/staging/03-building-a-streaming-pipeline/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,145 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable", + "freeShippingTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } + + @Test + public void createFreeShippingTable_shouldSendTheExpectedSQL() { + TableResult result = service.createFreeShippingTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("freeShippingTable")); + assertTrue(sql.contains("details")); + assertTrue(sql.toLowerCase().contains("row")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("product_id")); + assertTrue(sql.contains("price")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrdersOver50Dollars_shouldStreamTheExpectedRecordsToTheTable() { + TableResult result = service.streamOrdersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(2, expressions.length); + assertEquals("order_id", expressions[0].asSummaryString()); + assertEquals("as(row(customer_id, product_id, price), 'details')", expressions[1].asSummaryString()); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "freeShippingTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } +} diff --git a/staging/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java b/staging/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java new file mode 100644 index 0000000..e8c4467 --- /dev/null +++ b/staging/04-windowing/src/test/java/marketplace/OrderServiceIntegrationTest.java @@ -0,0 +1,365 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.*; + +@Tag("IntegrationTest") +class OrderServiceIntegrationTest extends FlinkIntegrationTest { + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderQualifiedForFreeShippingTableName = "`flink-table-api-java`.`marketplace`.`order-qualified-for-free-shipping-temp`"; + private final String orderQualifiedForFreeShippingShortTableName = "order-qualified-for-free-shipping-temp"; + private final String customerOrdersForPeriodTableName = "`flink-table-api-java`.`marketplace`.`customer-orders-collected-for-period-temp`"; + private final String customerOrdersForPeriodShortTableName = "customer-orders-collected-for-period-temp"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price"); + private Integer indexOf(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + + private OrderService orderService; + + private Row toQualifiedForFreeShippingRow(Row row) { + return Row.of( + row.getFieldAs(indexOf("order_id")), + Row.of( + row.getFieldAs(indexOf("customer_id")), + row.getFieldAs(indexOf("product_id")), + row.getFieldAs(indexOf("price")) + ) + ); + } + + @Override + public void setup() { + orderService = new OrderService( + env, + ordersTableName, + orderQualifiedForFreeShippingTableName, + customerOrdersForPeriodTableName + ); + } + + @Test + @Timeout(90) + public void ordersOver50Dollars_shouldOnlyReturnOrdersWithAPriceOf50DollarsOrMore() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create a set of orders with fixed prices + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.ordersOver50Dollars()); + + // Build the expected results. + List expected = orders.stream().filter(row -> row.getFieldAs(indexOf("price")) >= 50).toList(); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(expected.size()) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "customer_id", "product_id", "price" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(90) + public void pricesWithTax_shouldReturnTheCorrectPrices() { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + BigDecimal taxAmount = BigDecimal.valueOf(1.15); + + // Everything except 1 and 10.0 will result in a floating point precision issue. + Double[] prices = new Double[] { 1d, 65.30d, 10.0d, 95.70d, 35.25d }; + + // Create the orders. + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Execute the query. + TableResult results = retry(() -> orderService.pricesWithTax(taxAmount)); + + // Fetch the actual results. + List actual = fetchRows(results) + .limit(orders.size()) + .toList(); + + // Build the expected results. + List expected = orders.stream().map(row -> { + BigDecimal originalPrice = BigDecimal.valueOf(row.getFieldAs(indexOf("price"))) + .setScale(2, RoundingMode.HALF_UP); + BigDecimal priceWithTax = originalPrice + .multiply(taxAmount) + .setScale(2, RoundingMode.HALF_UP); + + return Row.of( + row.getFieldAs(indexOf("order_id")), + originalPrice, + priceWithTax + ); + }).toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + Set expectedFields = new HashSet<>(Arrays.asList( + "order_id", "original_price", "price_with_tax" + )); + assertTrue(actual.getFirst().getFieldNames(true).containsAll(expectedFields)); + } + + @Test + @Timeout(60) + public void createFreeShippingTable_shouldCreateTheTable() { + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + TableResult result = orderService.createFreeShippingTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderQualifiedForFreeShippingShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderQualifiedForFreeShippingShortTableName), + "Could not find the table: "+orderQualifiedForFreeShippingShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderQualifiedForFreeShippingShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrdersOver50Dollars_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(orderQualifiedForFreeShippingTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the destination table. + orderService.createFreeShippingTable().await(); + deleteTableOnExit(orderQualifiedForFreeShippingTableName); + + final int detailsPosition = 1; + + // Create a list of orders with specific prices. + Double[] prices = new Double[] { 25d, 49d, 50d, 51d, 75d }; + + List orders = Arrays.stream(prices).map(price -> + new OrderBuilder().withPrice(price).build() + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Initiate the stream. + cancelOnExit(orderService.streamOrdersOver50Dollars()); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderQualifiedForFreeShippingTableName) + .select($("*")) + .execute() + ); + + // Obtain the actual results. + List actual = fetchRows(queryResult) + .limit(Arrays.stream(prices).filter(p -> p >= 50).count()) + .toList(); + + // Build the expected results + List expected = orders.stream() + .filter(row -> row.getFieldAs(indexOf("price")) >= 50) + .map(this::toQualifiedForFreeShippingRow) + .toList(); + + // Assert on the results. + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + + assertEquals( + new HashSet<>(Arrays.asList("order_id", "details")), + actual.getFirst().getFieldNames(true) + ); + + assertEquals( + new HashSet<>(Arrays.asList("customer_id", "product_id", "price")), + actual.getFirst().getFieldAs(detailsPosition).getFieldNames(true) + ); + } + + @Test + @Timeout(60) + public void createOrdersForPeriodTable_shouldCreateTheTable() { + deleteTableOnExit(customerOrdersForPeriodTableName); + + TableResult result = orderService.createOrdersForPeriodTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+customerOrdersForPeriodShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(customerOrdersForPeriodShortTableName), + "Could not find the table: "+customerOrdersForPeriodShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE "+customerOrdersForPeriodTableName + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamCustomerPurchasesDuringPeriod_shouldStreamRecordsToTheTable() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(ordersTableName); + deleteTable(customerOrdersForPeriodTableName); + + // Create a temporary orders table. + createTemporaryTable(ordersTableName, ordersTableDefinition); + + // Create the output table. + orderService.createOrdersForPeriodTable().await(); + deleteTableOnExit(customerOrdersForPeriodTableName); + + final Duration windowSize = Duration.ofSeconds(10); + + // Create a set of products for each customer. + Map> customerProducts = new HashMap<>(); + customerProducts.put(1, Arrays.asList("Product1", "Product2")); + customerProducts.put(2, Arrays.asList("Product3", "Product4")); + customerProducts.put(3, Arrays.asList("Product5")); + customerProducts.put(4, Arrays.asList("Product6", "Product7")); + customerProducts.put(5, Arrays.asList("Product8", "Product9","Product9")); // Product9 is duplicated. + + // Create an order for each product. + List orders = customerProducts.entrySet().stream() + .flatMap(entry -> + entry.getValue().stream().map(v -> + new OrderBuilder() + .withCustomerId(entry.getKey()) + .withProductId(v) + .withTimestamp(Instant.now().truncatedTo(ChronoUnit.MILLIS).minus(windowSize)) + .build() + ) + ).toList(); + + // Push the orders into the temporary table. + env.fromValues(orders).insertInto(ordersTableName).execute(); + + // Flink only evaluates windows when a new event arrives. + // Therefore, in order to trigger the window closing, we need to issue + // another event after the window period. Because we are specifying the + // other events to occur 10 seconds early, just issuing an event now + // will do the trick. + env.fromValues(new OrderBuilder().build()).insertInto(ordersTableName).execute(); + + // Execute the method being tested. + cancelOnExit(orderService.streamOrdersForPeriod(windowSize)); + + // Fetch the results. + TableResult results = retry(() -> + env.from(customerOrdersForPeriodTableName) + .select($("*")) + .execute() + ); + + List actual = fetchRows(results) + .limit(customerProducts.size()).toList(); + + // Assert on the results. + actual.forEach(row -> { + Integer customerId = row.getFieldAs("customer_id"); + + Map actualProducts = row.getFieldAs("product_ids"); + Map expectedProducts = customerProducts.get(customerId).stream() + .collect(Collectors.groupingBy(product -> product, Collectors.summingInt(x -> 1))); + + assertEquals(expectedProducts, actualProducts); + assertEquals( + row.getFieldAs("window_start"), + row.getFieldAs("window_end").minus(windowSize) + ); + assertEquals(windowSize.toSeconds(), row.getFieldAs("period_in_seconds")); + }); + } +} \ No newline at end of file diff --git a/staging/04-windowing/src/test/java/marketplace/OrderServiceTest.java b/staging/04-windowing/src/test/java/marketplace/OrderServiceTest.java new file mode 100644 index 0000000..01429cd --- /dev/null +++ b/staging/04-windowing/src/test/java/marketplace/OrderServiceTest.java @@ -0,0 +1,215 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.math.BigDecimal; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.lit; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +@Tag("UnitTest") +public class OrderServiceTest { + private OrderService service; + private TableEnvironment mockEnv; + private Table mockTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + private GroupWindowedTable mockGroupWindowedTable; + private WindowGroupedTable mockWindowGroupedTable; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + mockGroupWindowedTable = mock(GroupWindowedTable.class); + mockWindowGroupedTable = mock(WindowGroupedTable.class); + service = new OrderService( + mockEnv, + "orderTable", + "freeShippingTable", + "ordersForPeriodTable" + ); + + when(mockEnv.from(anyString())).thenReturn(mockTable); + when(mockTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.window(any(GroupWindow.class))).thenReturn(mockGroupWindowedTable); + when(mockGroupWindowedTable.groupBy(any(Expression[].class))).thenReturn(mockWindowGroupedTable); + when(mockWindowGroupedTable.select(any(Expression[].class))).thenReturn(mockTable); + when(mockTable.where(any())).thenReturn(mockTable); + when(mockTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockTable.execute()).thenReturn(mockResult); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void ordersOver50Dollars_shouldSelectOrdersWhereThePriceIsGreaterThan50() { + TableResult result = service.ordersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + verify(mockTable).select(ArgumentMatchers.argThat(arg-> + arg.asSummaryString().equals("*") + )); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + assertEquals(mockResult, result); + } + + @Test + public void pricesWithTax_shouldReturnTheRecordIncludingThePriceWithTax() { + TableResult result = service.pricesWithTax(BigDecimal.valueOf(1.10)); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + List selectArgs = Arrays.stream(selectCaptor.getValue()).map(exp -> exp.asSummaryString()).toList(); + assertArrayEquals(new String[] { + "order_id", + "as(cast(price, DECIMAL(10, 2)), 'original_price')", + "as(round(times(cast(price, DECIMAL(10, 2)), 1.1), 2), 'price_with_tax')" + }, + selectArgs.toArray() + ); + + assertEquals(mockResult, result); + } + + @Test + public void createFreeShippingTable_shouldSendTheExpectedSQL() { + TableResult result = service.createFreeShippingTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("freeShippingTable")); + assertTrue(sql.contains("details")); + assertTrue(sql.toLowerCase().contains("row")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("product_id")); + assertTrue(sql.contains("price")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrdersOver50Dollars_shouldStreamTheExpectedRecordsToTheTable() { + TableResult result = service.streamOrdersOver50Dollars(); + + verify(mockEnv).from("orderTable"); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(2, expressions.length); + assertEquals("order_id", expressions[0].asSummaryString()); + assertEquals("as(row(customer_id, product_id, price), 'details')", expressions[1].asSummaryString()); + + verify(mockTable).where(ArgumentMatchers.argThat(arg -> + arg.asSummaryString().equals("greaterThanOrEqual(price, 50)") + )); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "freeShippingTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } + + @Test + public void createOrdersForPeriodTable_shouldSendTheExpectedSQL() { + TableResult result = service.createOrdersForPeriodTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("ordersForPeriodTable")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("window_start")); + assertTrue(sql.contains("window_end")); + assertTrue(sql.contains("period_in_seconds")); + assertTrue(sql.contains("product_ids")); + + assertEquals(mockResult, result); + } + + @Test + public void streamCustomerPurchasesDuringPeriod_shouldStreamTheExpectedRecordsToTheTable() { + Duration windowSize = Duration.ofSeconds(10); + TableResult result = service.streamOrdersForPeriod(windowSize); + + verify(mockEnv).useCatalog("examples"); + verify(mockEnv).useDatabase("marketplace"); + verify(mockEnv).from("orderTable"); + + ArgumentCaptor windowCaptor = ArgumentCaptor.forClass(GroupWindow.class); + verify(mockTable).window(windowCaptor.capture()); + + TumbleWithSizeOnTimeWithAlias window = (TumbleWithSizeOnTimeWithAlias) windowCaptor.getValue(); + String windowAlias = window.getAlias().asSummaryString(); + String timeField = window.getTimeField().asSummaryString(); + long size = Long.parseLong(window.getSize().asSummaryString()); + + assertEquals("$rowtime", timeField); + assertEquals(10000l, size); + + ArgumentCaptor groupByCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockGroupWindowedTable).groupBy(groupByCaptor.capture()); + Expression[] groupByExpressions = groupByCaptor.getValue(); + + assertEquals(2, groupByExpressions.length); + assertEquals("customer_id", groupByExpressions[0].asSummaryString()); + assertEquals(windowAlias, groupByExpressions[1].asSummaryString()); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockWindowGroupedTable).select(selectCaptor.capture()); + Expression[] expressions = selectCaptor.getValue(); + assertEquals(5, expressions.length); + assertEquals("customer_id", expressions[0].asSummaryString()); + assertEquals("start("+windowAlias+")", expressions[1].asSummaryString()); + assertEquals("end("+windowAlias+")", expressions[2].asSummaryString()); + assertEquals("as("+lit(windowSize.getSeconds()).seconds().asSummaryString()+", 'period_in_seconds')" , expressions[3].asSummaryString()); + assertEquals("as(collect(product_id), 'product_ids')", expressions[4].asSummaryString()); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockTable).insertInto(insertCaptor.capture()); + assertEquals( + "ordersForPeriodTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } +} diff --git a/staging/05-joins/pom.xml b/staging/05-joins/pom.xml new file mode 100644 index 0000000..d0bde22 --- /dev/null +++ b/staging/05-joins/pom.xml @@ -0,0 +1,258 @@ + + + 4.0.0 + + marketplace + flink-table-api-marketplace + 0.1 + jar + + Flink Table API Marketplace on Confluent Cloud + + + UTF-8 + 1.20.0 + 0.92.0 + 3.8.0 + 7.7.0 + 21 + ${target.java.version} + ${target.java.version} + 2.17.1 + + + + + plugins + file://${project.basedir}/../plugins + + + solution.plugins + file://${project.basedir}/../../plugins + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + confluent + https://packages.confluent.io/maven/ + + + + + + + org.apache.flink + flink-table-api-java + ${flink.version} + + + + + io.confluent.flink + confluent-table-planner + ${confluent-plugin.version} + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + test + + + io.confluent + kafka-schema-registry-client + ${schema-registry-client.version} + test + + + + org.junit.jupiter + junit-jupiter-engine + 5.9.2 + test + + + org.junit.platform + junit-platform-runner + 1.9.2 + test + + + org.mockito + mockito-core + 5.12.0 + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + marketplace.Marketplace + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + org.junit.platform + junit-platform-surefire-provider + 1.2.0 + + + + + src/test/java/ + + -XX:+EnableDynamicAgentLoading + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + diff --git a/staging/05-joins/src/main/java/marketplace/.gitignore b/staging/05-joins/src/main/java/marketplace/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/staging/05-joins/src/main/java/marketplace/ClickService.java b/staging/05-joins/src/main/java/marketplace/ClickService.java new file mode 100644 index 0000000..4c8a435 --- /dev/null +++ b/staging/05-joins/src/main/java/marketplace/ClickService.java @@ -0,0 +1,38 @@ +package marketplace; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; + +import java.time.Duration; + +import static org.apache.flink.table.api.Expressions.*; + +public class ClickService { + private final TableEnvironment env; + private final String clicksTableName; + private final String ordersTableName; + private final String orderPlacedAfterClickTableName; + + public ClickService( + TableEnvironment env, + String clicksTableName, + String ordersTableName, + String orderPlacedAfterClickTableName + ) { + this.env = env; + this.clicksTableName = clicksTableName; + this.ordersTableName = ordersTableName; + this.orderPlacedAfterClickTableName = orderPlacedAfterClickTableName; + } + + public TableResult createOrderPlacedAfterClickTable() { + // TODO + return null; + } + + public TableResult streamOrderPlacedAfterClick(Duration withinTimePeriod) { + // TODO + return null; + } +} diff --git a/staging/05-joins/src/test/java/marketplace/ClickBuilder.java b/staging/05-joins/src/test/java/marketplace/ClickBuilder.java new file mode 100644 index 0000000..0b47178 --- /dev/null +++ b/staging/05-joins/src/test/java/marketplace/ClickBuilder.java @@ -0,0 +1,61 @@ +package marketplace; + +import org.apache.flink.types.Row; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +public class ClickBuilder { + private String clickId; + private int userId; + private String url; + private String userAgent; + private int viewTime; + private Instant timestamp; + + public ClickBuilder() { + Random rnd = new Random(); + + clickId = "Click"+rnd.nextInt(1000); + userId = rnd.nextInt(1000); + url = "http://some.url/"+rnd.nextInt(1000); + userAgent = "UserAgent"+rnd.nextInt(1000); + viewTime = rnd.nextInt(1000); + timestamp = Instant.now().truncatedTo( ChronoUnit.MILLIS ); + } + + public ClickBuilder withClickId(String clickId) { + this.clickId = clickId; + return this; + } + + public ClickBuilder withUserId(int userId) { + this.userId = userId; + return this; + } + + public ClickBuilder withUrl(String url) { + this.url = url; + return this; + } + + public ClickBuilder withViewTime(int viewTime) { + this.viewTime = viewTime; + return this; + } + + public ClickBuilder withUserAgent(String user_agent) { + this.userAgent = user_agent; + return this; + } + + public ClickBuilder withTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public Row build() { + return Row.of(clickId, userId, url, userAgent, viewTime, timestamp); + } +} diff --git a/staging/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java b/staging/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java new file mode 100644 index 0000000..fb691ae --- /dev/null +++ b/staging/05-joins/src/test/java/marketplace/ClickServiceIntegrationTest.java @@ -0,0 +1,241 @@ +package marketplace; + +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +import static org.apache.flink.table.api.Expressions.$; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("IntegrationTest") +class ClickServiceIntegrationTest extends FlinkIntegrationTest { + private final String clicksTableName = "`flink-table-api-java`.`marketplace`.`clicks-temp`"; + private final String ordersTableName = "`flink-table-api-java`.`marketplace`.`orders-temp`"; + private final String orderPlacedAfterClickTableName = "`flink-table-api-java`.`marketplace`.`order-placed-after-click-temp`"; + private final String orderPlacedAfterClickShortTableName = "order-placed-after-click-temp"; + + private final String clicksTableDefinition = + "CREATE TABLE IF NOT EXISTS " + clicksTableName + " (\n" + + " `click_id` VARCHAR(2147483647) NOT NULL,\n" + + " `user_id` INT NOT NULL,\n" + + " `url` VARCHAR(2147483647) NOT NULL,\n" + + " `user_agent` VARCHAR(2147483647) NOT NULL,\n" + + " `view_time` INT NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final String ordersTableDefinition = + "CREATE TABLE IF NOT EXISTS " + ordersTableName + " (\n" + + " `order_id` VARCHAR(2147483647) NOT NULL,\n" + + " `customer_id` INT NOT NULL,\n" + + " `product_id` VARCHAR(2147483647) NOT NULL,\n" + + " `price` DOUBLE NOT NULL,\n" + + " `event_time` TIMESTAMP_LTZ(3) METADATA FROM 'timestamp',\n" + + " `$rowtime` TIMESTAMP_LTZ(3) NOT NULL METADATA VIRTUAL COMMENT 'SYSTEM',\n" + + " WATERMARK FOR `$rowtime` AS `$rowtime`\n" + + ") DISTRIBUTED INTO 1 BUCKETS WITH (\n" + + " 'kafka.retention.time' = '1 h',\n" + + " 'scan.startup.mode' = 'earliest-offset'\n" + + ");"; + + private final List orderTableFields = Arrays.asList("order_id", "customer_id", "product_id", "price", "event_time"); + private Integer indexOfOrderField(String fieldName) { + return orderTableFields.indexOf(fieldName); + } + private final List clickTableFields = Arrays.asList("click_id", "user_id", "url", "user_agent", "view_time", "event_time"); + private Integer indexOfClickField(String fieldName) { + return clickTableFields.indexOf(fieldName); + } + + private ClickService clickService; + + @Override + public void setup() { + super.setup(); + clickService = new ClickService( + env, + clicksTableName, + ordersTableName, + orderPlacedAfterClickTableName + ); + } + + @Test + @Timeout(60) + public void createOrderPlacedAfterClickTable_shouldCreateTheTable() { + deleteTableOnExit(orderPlacedAfterClickTableName); + + TableResult result = clickService.createOrderPlacedAfterClickTable(); + + String status = result.collect().next().getFieldAs(0); + assertEquals("Table '"+orderPlacedAfterClickShortTableName+"' created", status); + + env.useCatalog("flink-table-api-java"); + env.useDatabase("marketplace"); + String[] tables = env.listTables(); + assertTrue( + Arrays.asList(tables).contains(orderPlacedAfterClickShortTableName), + "Could not find the table: "+orderPlacedAfterClickShortTableName + ); + + String tableDefinition = env.executeSql( + "SHOW CREATE TABLE `"+orderPlacedAfterClickShortTableName+"`" + ).collect().next().getFieldAs(0); + + assertTrue( + tableDefinition.contains("'connector' = 'confluent',"), + "Incorrect connector. Expected 'confluent'" + ); + assertTrue( + tableDefinition.contains("'scan.startup.mode' = 'earliest-offset'"), + "Incorrect scan.startup.mode. Expected 'earliest-offset'" + ); + } + + @Test + @Timeout(180) + public void streamOrderPlacedAfterClick_shouldJoinOrdersAndClicksAndEmitANewStream() throws Exception { + // Clean up any tables left over from previously executing this test. + deleteTable(clicksTableName); + deleteTable(ordersTableName); + deleteTable(orderPlacedAfterClickTableName); + + // Create the necessary tables. + createTemporaryTable(clicksTableName, clicksTableDefinition); + createTemporaryTable(ordersTableName, ordersTableDefinition); + clickService.createOrderPlacedAfterClickTable().await(); + deleteTableOnExit(orderPlacedAfterClickTableName); + + // Define some constants. + final Duration withinTimePeriod = Duration.ofMinutes(5); + final Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); + final Instant onTime = now.plusSeconds(1); + final Instant late = now.plus(withinTimePeriod).plusSeconds(1); + + // Define some customer Ids. + List customerIds = Arrays.asList(1, 2, 3, 4, 5); + + // Create some clicks. + List expectedClicks = customerIds.stream() + .map(customer -> + new ClickBuilder() + .withUserId(customer) + .withTimestamp(now) + .build() + ) + .toList(); + + // Mutable copy of the clicks. + List onTimeClicks = new ArrayList<>(expectedClicks); + + // Add a non-matching user Id. + onTimeClicks.add( + new ClickBuilder() + .withUserId(99) + .withTimestamp(now) + .build() + ); + + // Randomize the list. + Collections.shuffle(onTimeClicks); + + // Create some orders. + List expectedOrders = customerIds.stream() + .map(customer -> + new OrderBuilder() + .withCustomerId(customer) + .withTimestamp(onTime) + .build() + ) + .toList(); + + // Mutable copy of the orders + List onTimeOrders = new ArrayList<>(expectedOrders); + + // Add a non-matching customer Id. + onTimeOrders.add( + new OrderBuilder() + .withCustomerId(101) + .withTimestamp(onTime) + .build() + ); + + // Randomize the list. + Collections.shuffle(onTimeOrders); + + // Create a late click. + Row lateClick = new ClickBuilder() + .withUserId(1) + .withTimestamp(late) + .build(); + + // Create a late order. + Row lateOrder = new OrderBuilder() + .withCustomerId(5) + .withTimestamp(late) + .build(); + + // Push data into the destination tables. + env.fromValues(onTimeClicks).insertInto(clicksTableName).execute(); + env.fromValues(onTimeOrders).insertInto(ordersTableName).execute(); + + // We push the late data separately, to ensure it actually comes after the earlier data. + env.fromValues(lateClick).insertInto(clicksTableName).execute(); + env.fromValues(lateOrder).insertInto(ordersTableName).execute(); + + // Execute the query we are testing. + cancelOnExit(clickService.streamOrderPlacedAfterClick(withinTimePeriod)); + + // Query the destination table. + TableResult queryResult = retry(() -> + env.from(orderPlacedAfterClickTableName) + .select($("*")) + .execute() + ); + + Set actual = new HashSet<>( + fetchRows(queryResult) + .limit(customerIds.size()) + .toList() + ); + + // Build the expected results. + Set expected = new HashSet<> ( + customerIds.stream().map(customer -> { + Row clickRow = expectedClicks.stream() + .filter(click -> click.getFieldAs(indexOfClickField("user_id")).equals(customer)) + .findFirst() + .orElseThrow(); + + Row orderRow = expectedOrders.stream() + .filter(click -> click.getFieldAs(indexOfOrderField("customer_id")).equals(customer)) + .findFirst() + .orElseThrow(); + + return Row.of( + customer, + clickRow.getField(indexOfClickField("url")), + clickRow.getField(indexOfClickField("event_time")), + orderRow.getField(indexOfOrderField("product_id")), + orderRow.getField(indexOfOrderField("event_time")) + ); + }).toList() + ); + + // Assert on the results. + assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/staging/05-joins/src/test/java/marketplace/ClickServiceTest.java b/staging/05-joins/src/test/java/marketplace/ClickServiceTest.java new file mode 100644 index 0000000..fb6d809 --- /dev/null +++ b/staging/05-joins/src/test/java/marketplace/ClickServiceTest.java @@ -0,0 +1,121 @@ +package marketplace; + +import org.apache.flink.table.api.*; +import org.apache.flink.table.expressions.Expression; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; + +class ClickServiceTest { + private ClickService service; + private TableEnvironment mockEnv; + private Table mockClicksTable; + private Table mockOrdersTable; + private Table mockJoinedTable; + private TableResult mockResult; + private TablePipeline mockPipeline; + + @BeforeEach + public void setup() { + mockEnv = mock(TableEnvironment.class); + mockClicksTable = mock(Table.class); + mockOrdersTable = mock(Table.class); + mockJoinedTable = mock(Table.class); + mockPipeline = mock(TablePipeline.class); + mockResult = mock(TableResult.class); + service = new ClickService( + mockEnv, + "clickTable", + "orderTable", + "orderPlacedAfterClickTable" + ); + + when(mockEnv.from("clickTable")).thenReturn(mockClicksTable); + when(mockClicksTable.select(any(Expression[].class))).thenReturn(mockClicksTable); + when(mockEnv.from("orderTable")).thenReturn(mockOrdersTable); + when(mockOrdersTable.select(any(Expression[].class))).thenReturn(mockOrdersTable); + when(mockClicksTable.join(any())).thenReturn(mockJoinedTable); + when(mockOrdersTable.join(any())).thenReturn(mockJoinedTable); + when(mockJoinedTable.where(any())).thenReturn(mockJoinedTable); + when(mockJoinedTable.select(any(Expression[].class))).thenReturn(mockJoinedTable); + when(mockJoinedTable.insertInto(anyString())).thenReturn(mockPipeline); + when(mockPipeline.execute()).thenReturn(mockResult); + when(mockEnv.executeSql(anyString())).thenReturn(mockResult); + } + + @Test + public void createOrderPlacedAfterClickTable_shouldSendTheExpectedSQL() { + TableResult result = service.createOrderPlacedAfterClickTable(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + + verify(mockEnv).executeSql(captor.capture()); + + String sql = captor.getValue(); + + assertTrue(sql.toLowerCase().contains("create table if not exists")); + assertTrue(sql.contains("orderPlacedAfterClickTable")); + assertTrue(sql.contains("customer_id")); + assertTrue(sql.contains("clicked_url")); + assertTrue(sql.contains("time_of_click")); + assertTrue(sql.contains("purchased_product")); + assertTrue(sql.contains("time_of_order")); + assertTrue(sql.contains("scan.startup.mode")); + assertTrue(sql.contains("earliest-offset")); + + assertEquals(mockResult, result); + } + + @Test + public void streamOrderPlacedAfterClick_shouldStreamTheExpectedRecordsToTheTable() { + Duration withinTimePeriod = Duration.ofMinutes(5); + TableResult result = service.streamOrderPlacedAfterClick(withinTimePeriod); + + verify(mockEnv).from("clickTable"); + verify(mockEnv).from("orderTable"); + + ArgumentCaptor whereCaptor = ArgumentCaptor.forClass(Expression.class); + verify(mockJoinedTable).where(whereCaptor.capture()); + String whereExpression = whereCaptor.getValue().asSummaryString(); + + assertTrue(whereExpression.contains("and")); + assertTrue( + whereExpression.contains("equals(user_id, customer_id") + || whereExpression.contains("equals(customer_id, user_id)") + ); + assertTrue(whereExpression.contains(Long.toString(withinTimePeriod.toSeconds()))); + + ArgumentCaptor selectCaptor = ArgumentCaptor.forClass(Expression[].class); + verify(mockJoinedTable).select(selectCaptor.capture()); + List selectExpressions = Arrays.stream(selectCaptor.getValue()) + .map(Expression::asSummaryString) + .toList(); + + assertTrue(selectExpressions.get(0).contains("customer_id")); + assertTrue(selectExpressions.get(1).contains("clicked_url")); + assertTrue(selectExpressions.get(2).contains("time_of_click")); + assertTrue(selectExpressions.get(3).contains("purchased_product")); + assertTrue(selectExpressions.get(4).contains("time_of_order")); + + ArgumentCaptor insertCaptor = ArgumentCaptor.forClass(String.class); + verify(mockJoinedTable).insertInto(insertCaptor.capture()); + + assertEquals( + "orderPlacedAfterClickTable", + insertCaptor.getValue().replace("`marketplace`", "marketplace") + ); + + assertEquals(mockResult, result); + } + +} \ No newline at end of file