Skip to content
Permalink
Browse files
[FLINK-20598] [training] Update playgrounds for Flink 1.12 (#21) 
* Update ops-playground to v1.12.1

* Create local dirs for the checkpoints&savepoints before spinning up the docker compose application.

* Upgrade table-walkthrough with flink-1.12.1 version.

* Upgrade pyflink-walkthrough with flink-1.12.1 version.

* [FLINK-20599] Change flink-playground-clickcountjob version from 1.11.0 to 1.12.1.

* Update operations-playground/README.md

Co-authored-by: David Anderson <david@alpinegizmo.com>
  • Loading branch information
@victorunique @alpinegizmo
victorunique and alpinegizmo committed Jan 27, 2021
1 parent 74e4c98 commit 05680d1a6a1014528c65b49c97baa4f09d0987ea
Show file tree
Hide file tree
Showing 10 changed files with 33 additions and 26 deletions.
2 docker/ops-playground-image/Dockerfile
View file
@@ -32,7 +32,7 @@ RUN mvn clean install
# Build Operations Playground Image
###############################################################################

FROM flink:1.11.0-scala_2.11
FROM apache/flink:1.12.1-scala_2.11-java8

WORKDIR /opt/flink/bin

4 docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml
View file
@@ -22,7 +22,7 @@ under the License.

<groupId>org.apache.flink</groupId>
<artifactId>flink-playground-clickcountjob</artifactId>
<version>1-FLINK-1.11_2.11</version>
<version>1-FLINK-1.12_2.11</version>

<name>flink-playground-clickcountjob</name>
<packaging>jar</packaging>
@@ -44,7 +44,7 @@ under the License.

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.11.0</flink.version>
<flink.version>1.12.1</flink.version>
<java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
11 operations-playground/README.md
View file
@@ -26,6 +26,15 @@ Build the Docker image by running
docker-compose build
```

### Preparing the Checkpoint and Savepoint Directories

Create the checkpoint and savepoint directories on the Docker host machine (these volumes are mounted by the jobmanager and taskmanager, as specified in docker-compose.yaml):

```bash
mkdir -p /tmp/flink-checkpoints-directory
mkdir -p /tmp/flink-savepoints-directory
```

### Starting the Playground

Once you built the Docker image, run the following command to start the playground
@@ -52,4 +61,4 @@ docker-compose down
## Further instructions

The playground setup and more detailed instructions are presented in the
["Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-release-1.11/try-flink/flink-operations-playground.html) of Flink's documentation.
["Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-release-1.12/try-flink/flink-operations-playground.html) of Flink's documentation.
14 operations-playground/docker-compose.yaml
View file
@@ -20,7 +20,7 @@ version: "2.1"
services:
client:
build: ../docker/ops-playground-image
image: apache/flink-ops-playground:1-FLINK-1.11-scala_2.11
image: apache/flink-ops-playground:1-FLINK-1.12-scala_2.11
command: "flink run -d /opt/ClickCountJob.jar --bootstrap.servers kafka:9092 --checkpointing --event-time"
depends_on:
- jobmanager
@@ -30,29 +30,29 @@ services:
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
clickevent-generator:
image: apache/flink-ops-playground:1-FLINK-1.11-scala_2.11
image: apache/flink-ops-playground:1-FLINK-1.12-scala_2.11
command: "java -classpath /opt/ClickCountJob.jar:/opt/flink/lib/* org.apache.flink.playgrounds.ops.clickcount.ClickEventGenerator --bootstrap.servers kafka:9092 --topic input"
depends_on:
- kafka
jobmanager:
image: flink:1.11.0-scala_2.11
image: apache/flink:1.12.1-scala_2.11-java8
command: "jobmanager.sh start-foreground"
ports:
- 8081:8081
volumes:
- ./conf:/opt/flink/conf
- flink-checkpoints-directory:/tmp/flink-checkpoints-directory
- /tmp/flink-checkpoints-directory:/tmp/flink-checkpoints-directory
- /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
taskmanager:
image: flink:1.11.0-scala_2.11
image: apache/flink:1.12.1-scala_2.11-java8
depends_on:
- jobmanager
command: "taskmanager.sh start-foreground"
volumes:
- ./conf:/opt/flink/conf
- flink-checkpoints-directory:/tmp/flink-checkpoints-directory
- /tmp/flink-checkpoints-directory:/tmp/flink-checkpoints-directory
- /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
@@ -69,5 +69,3 @@ services:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ports:
- 9094:9094
volumes:
flink-checkpoints-directory:
6 pyflink-walkthrough/Dockerfile
View file
@@ -20,8 +20,8 @@
# Build PyFlink Playground Image
###############################################################################

FROM flink:1.11.0-scala_2.11
ARG FLINK_VERSION=1.11.1
FROM apache/flink:1.12.1-scala_2.11-java8
ARG FLINK_VERSION=1.12.1

# Install pyflink
RUN set -ex; \
@@ -33,7 +33,7 @@ RUN set -ex; \
ln -s /usr/bin/pip3 /usr/bin/pip; \
apt-get update; \
python -m pip install --upgrade pip; \
pip install apache-flink==1.11.1; \
pip install apache-flink==1.12.1; \
pip install kafka-python;


4 pyflink-walkthrough/README.md
View file
@@ -5,7 +5,7 @@
In this playground, you will learn how to build and run an end-to-end PyFlink pipeline for data analytics, covering the following steps:

* Reading data from a Kafka source;
* Creating data using a [UDF](https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/python/table-api-users-guide/udfs/python_udfs.html);
* Creating data using a [UDF](https://ci.apache.org/projects/flink/flink-docs-release-1.12/dev/python/table-api-users-guide/udfs/python_udfs.html);
* Performing a simple aggregation over the source data;
* Writing the results to Elasticsearch and visualizing them in Kibana.

@@ -153,4 +153,4 @@ $ docker-compose exec jobmanager ./bin/flink run -py /opt/pyflink-walkthrough/pa
```

Furthermore, you can also [create new Kibana dashboards](https://www.elastic.co/guide/en/kibana/7.8/dashboard-create-new-dashboard.html)
to visualize other aspects of the data in Elasticsearch.
to visualize other aspects of the data in Elasticsearch.
4 pyflink-walkthrough/docker-compose.yml
View file
@@ -20,7 +20,7 @@ version: '2.1'
services:
jobmanager:
build: .
image: pyflink/pyflink:1.11.0-scala_2.11
image: pyflink/pyflink:1.12.1-scala_2.11
volumes:
- .:/opt/pyflink-walkthrough
hostname: "jobmanager"
@@ -32,7 +32,7 @@ services:
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
taskmanager:
image: pyflink/pyflink:1.11.0-scala_2.11
image: pyflink/pyflink:1.12.1-scala_2.11
volumes:
- .:/opt/pyflink-walkthrough
expose:
8 table-walkthrough/Dockerfile
View file
@@ -22,12 +22,12 @@ COPY ./pom.xml /opt/pom.xml
COPY ./src /opt/src
RUN cd /opt; mvn clean install -Dmaven.test.skip

FROM flink:1.11.0-scala_2.11
FROM apache/flink:1.12.1-scala_2.11-java8

# Download connector libraries
RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka_2.11/1.11.0/flink-sql-connector-kafka_2.11-1.11.0.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc_2.11/1.11.0/flink-connector-jdbc_2.11-1.11.0.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/1.11.0/flink-csv-1.11.0.jar; \
RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka_2.11/1.12.1/flink-sql-connector-kafka_2.11-1.12.1.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc_2.11/1.12.1/flink-connector-jdbc_2.11-1.12.1.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/1.12.1/flink-csv-1.12.1.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.19/mysql-connector-java-8.0.19.jar;

COPY --from=builder /opt/target/spend-report-*.jar /opt/flink/usrlib/spend-report.jar
4 table-walkthrough/docker-compose.yml
View file
@@ -19,7 +19,7 @@
version: '2.1'
services:
jobmanager:
image: apache/flink-table-walkthrough:1-FLINK-1.11-scala_2.11
image: apache/flink-table-walkthrough:1-FLINK-1.12-scala_2.11
build: .
hostname: "jobmanager"
expose:
@@ -33,7 +33,7 @@ services:
- kafka
- mysql
taskmanager:
image: apache/flink-table-walkthrough:1-FLINK-1.11-scala_2.11
image: apache/flink-table-walkthrough:1-FLINK-1.12-scala_2.11
build: .
expose:
- "6121"
2 table-walkthrough/pom.xml
View file
@@ -30,7 +30,7 @@ under the License.

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.11.0</flink.version>
<flink.version>1.12.1</flink.version>
<java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>

0 comments on commit 05680d1

Please sign in to comment.