From c68a138c94171fa36341dd723502dbd8aaf85777 Mon Sep 17 00:00:00 2001 From: Joao Boto Date: Tue, 11 Dec 2018 22:23:55 +0100 Subject: [PATCH] [BAHIR-147] update with last extensions Closes #14 --- NOTICE | 2 +- .../_includes/themes/apache-clean/footer.html | 5 +- site/docs/flink/current/documentation.md | 8 +- .../flink/current/flink-streaming-activemq.md | 2 +- .../flink/current/flink-streaming-akka.md | 2 +- .../flink/current/flink-streaming-flume.md | 6 +- .../flink/current/flink-streaming-influxdb.md | 59 +++++++++ .../flink/current/flink-streaming-kudu.md | 125 ++++++++++++++++++ .../flink/current/flink-streaming-netty.md | 2 +- .../flink/current/flink-streaming-redis.md | 2 +- site/docs/flink/overview.md | 2 +- .../flink-streaming-influxdb.template | 27 ++++ .../templates/flink-streaming-kudu.template | 27 ++++ site/index.md | 4 +- 14 files changed, 258 insertions(+), 15 deletions(-) create mode 100644 site/docs/flink/current/flink-streaming-influxdb.md create mode 100644 site/docs/flink/current/flink-streaming-kudu.md create mode 100644 site/docs/flink/templates/flink-streaming-influxdb.template create mode 100644 site/docs/flink/templates/flink-streaming-kudu.template diff --git a/NOTICE b/NOTICE index 4127e40..7cf47a7 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Website Template -Copyright [2016-2017] The Apache Software Foundation +Copyright [2016-2018] The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/site/_includes/themes/apache-clean/footer.html b/site/_includes/themes/apache-clean/footer.html index 6070a43..f86e28d 100644 --- a/site/_includes/themes/apache-clean/footer.html +++ b/site/_includes/themes/apache-clean/footer.html @@ -18,9 +18,8 @@ {% endif %}
- Copyright © 2016-2017 The Apache Software Foundation. - Licensed under the Apache License, Version - 2.0. + Copyright © 2016- The Apache Software Foundation. + Licensed under the Apache License, Version 2.0.
{% if site.data.project.podling %} Apache, the Apache Feather logo, and the Apache Incubator project logo are trademarks of The Apache diff --git a/site/docs/flink/current/documentation.md b/site/docs/flink/current/documentation.md index 55b557f..1af2a3c 100644 --- a/site/docs/flink/current/documentation.md +++ b/site/docs/flink/current/documentation.md @@ -1,7 +1,7 @@ --- layout: page -title: Extensions for Apache Flink (1.0.0-SNAPSHOT) -description: Extensions for Apache Flink (1.0.0-SNAPSHOT) +title: Extensions for Apache Flink (1.1-SNAPSHOT) +description: Extensions for Apache Flink (1.1-SNAPSHOT) group: nav-right --- + +{% include JB/setup %} + +# Flink InfluxDB Connector + +This connector provides a sink that can send data to [InfluxDB](https://www.influxdata.com/). To use this connector, add the +following dependency to your project: + + + org.apache.bahir + flink-connector-influxdb_2.11 + 1.1-SNAPSHOT + + +*Version Compatibility*: This module is compatible with InfluxDB 1.3.x +*Requirements*: Java 1.8+ + +Note that the streaming connectors are not part of the binary distribution of Flink. You need to link them into your job jar for cluster execution. +See how to link with them for cluster execution [here](https://ci.apache.org/projects/flink/flink-docs-release-1.3/dev/linking.html). + +## Installing InfluxDB +Follow the instructions from the [InfluxDB download page](https://portal.influxdata.com/downloads#influxdb). + +## Examples + +### JAVA API + + DataStream dataStream = ... + InfluxDBConfig influxDBConfig = InfluxDBConfig.builder(String host, String username, String password, String dbName) + dataStream.addSink(new InfluxDBSink(influxDBConfig)); + + +See end-to-end examples at [InfluxDB Examples](https://github.com/apache/bahir-flink/tree/master/flink-connector-influxdb/examples) + + diff --git a/site/docs/flink/current/flink-streaming-kudu.md b/site/docs/flink/current/flink-streaming-kudu.md new file mode 100644 index 0000000..2eef38c --- /dev/null +++ b/site/docs/flink/current/flink-streaming-kudu.md @@ -0,0 +1,125 @@ +--- +layout: page +title: Apache Flink Streaming Connector for Apache Kudu +description: Apache Flink Streaming Connector for Apache Kudu +group: nav-right +--- + + +{% include JB/setup %} + +# Flink Kudu Connector + +This connector provides a source (```KuduInputFormat```) and a sink/output (```KuduSink``` and ```KuduOutputFormat```, respectively) that can read and write to [Kudu](https://kudu.apache.org/). To use this connector, add the +following dependency to your project: + + + org.apache.bahir + flink-connector-kudu_2.11 + 1.1-SNAPSHOT + + +*Version Compatibility*: This module is compatible with Apache Kudu *1.7.1* (last stable version). + +Note that the streaming connectors are not part of the binary distribution of Flink. You need to link them into your job jar for cluster execution. +See how to link with them for cluster execution [here](https://ci.apache.org/projects/flink/flink-docs-stable/start/dependencies.html). + +## Installing Kudu + +Follow the instructions from the [Kudu Installation Guide](https://kudu.apache.org/docs/installation.html). +Optionally, you can use the docker images provided in dockers folder. + +## KuduInputFormat + +``` +ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + +env.setParallelism(PARALLELISM); + +// create a table info object +KuduTableInfo tableInfo = KuduTableInfo.Builder + .create("books") + .addColumn(KuduColumnInfo.Builder.create("id", Type.INT32).key(true).hashKey(true).build()) + .addColumn(KuduColumnInfo.Builder.create("title", Type.STRING).build()) + .addColumn(KuduColumnInfo.Builder.create("author", Type.STRING).build()) + .addColumn(KuduColumnInfo.Builder.create("price", Type.DOUBLE).build()) + .addColumn(KuduColumnInfo.Builder.create("quantity", Type.INT32).build()) + .build(); + +// Pass the tableInfo to the KuduInputFormat and provide kuduMaster ips +env.createInput(new KuduInputFormat<>("172.25.0.6", tableInfo)) + .count(); + +env.execute(); +``` + +## KuduOutputFormat + +``` +ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + +env.setParallelism(PARALLELISM); + +// create a table info object +KuduTableInfo tableInfo = KuduTableInfo.Builder + .create("books") + .createIfNotExist(true) + .replicas(1) + .addColumn(KuduColumnInfo.Builder.create("id", Type.INT32).key(true).hashKey(true).build()) + .addColumn(KuduColumnInfo.Builder.create("title", Type.STRING).build()) + .addColumn(KuduColumnInfo.Builder.create("author", Type.STRING).build()) + .addColumn(KuduColumnInfo.Builder.create("price", Type.DOUBLE).build()) + .addColumn(KuduColumnInfo.Builder.create("quantity", Type.INT32).build()) + .build(); + +... + +env.fromCollection(books) + .output(new KuduOutputFormat<>("172.25.0.6", tableInfo)); + +env.execute(); +``` + +## KuduSink + +``` +StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + +env.setParallelism(PARALLELISM); + +// create a table info object +KuduTableInfo tableInfo = KuduTableInfo.Builder + .create("books") + .createIfNotExist(true) + .replicas(1) + .addColumn(KuduColumnInfo.Builder.create("id", Type.INT32).key(true).hashKey(true).build()) + .addColumn(KuduColumnInfo.Builder.create("title", Type.STRING).build()) + .addColumn(KuduColumnInfo.Builder.create("author", Type.STRING).build()) + .addColumn(KuduColumnInfo.Builder.create("price", Type.DOUBLE).build()) + .addColumn(KuduColumnInfo.Builder.create("quantity", Type.INT32).build()) + .build(); + +... + +env.fromCollection(books) + .addSink(new KuduSink<>("172.25.0.6", tableInfo)); + +env.execute(); +``` diff --git a/site/docs/flink/current/flink-streaming-netty.md b/site/docs/flink/current/flink-streaming-netty.md index 9ec2442..320537c 100644 --- a/site/docs/flink/current/flink-streaming-netty.md +++ b/site/docs/flink/current/flink-streaming-netty.md @@ -61,7 +61,7 @@ To use this connector, add the following dependency to your project: org.apache.bahir flink-connector-netty_2.11 - 1.0-SNAPSHOT + 1.1-SNAPSHOT ``` diff --git a/site/docs/flink/current/flink-streaming-redis.md b/site/docs/flink/current/flink-streaming-redis.md index 01db8c9..1a2f8c2 100644 --- a/site/docs/flink/current/flink-streaming-redis.md +++ b/site/docs/flink/current/flink-streaming-redis.md @@ -34,7 +34,7 @@ following dependency to your project: org.apache.bahir flink-connector-redis_2.11 - 1.0-SNAPSHOT + 1.1-SNAPSHOT *Version Compatibility*: This module is compatible with Redis 2.8.5. diff --git a/site/docs/flink/overview.md b/site/docs/flink/overview.md index 86a4b60..a6793b6 100644 --- a/site/docs/flink/overview.md +++ b/site/docs/flink/overview.md @@ -27,5 +27,5 @@ limitations under the License. ### Apache Bahir Extensions for Apache Flink - - [Current - 1.0-SNAPSHOT](/docs/flink/current/documentation) + - [Current - 1.1-SNAPSHOT](/docs/flink/current/documentation) - [1.0](/docs/flink/1.0/documentation) diff --git a/site/docs/flink/templates/flink-streaming-influxdb.template b/site/docs/flink/templates/flink-streaming-influxdb.template new file mode 100644 index 0000000..c507575 --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-influxdb.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for InfluxDB +description: Apache Flink Streaming Connector for InfluxDB +group: nav-right +--- + + +{% include JB/setup %} + diff --git a/site/docs/flink/templates/flink-streaming-kudu.template b/site/docs/flink/templates/flink-streaming-kudu.template new file mode 100644 index 0000000..53d007d --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-kudu.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for Apache Kudu +description: Apache Flink Streaming Connector for Apache Kudu +group: nav-right +--- + + +{% include JB/setup %} + diff --git a/site/index.md b/site/index.md index b105935..e6181d4 100644 --- a/site/index.md +++ b/site/index.md @@ -48,9 +48,11 @@ Currently, {{ site.data.project.short_name }} provides extensions for [Apache Sp ## Apache Flink extensions - Flink streaming connector for ActiveMQ + - Flink streaming connector for Akka - Flink streaming connector for Flume + - Flink streaming connector for InfluxDB ![](/assets/themes/apache-clean/img/new-black.png){:height="36px" width="36px"} + - Flink streaming connector for Kudu ![](/assets/themes/apache-clean/img/new-black.png){:height="36px" width="36px"} - Flink streaming connector for Redis - - Flink streaming connector for Akka - Flink streaming connector for Netty