Merge 888dfd5 into b0ba71f

epiphanous · Sep 10, 2020 · 449f61a · 449f61a
2 parents b0ba71f + 888dfd5
commit 449f61a
Show file tree

Hide file tree

Showing 40 changed files with 869 additions and 430 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,11 +1,12 @@
 language: scala
 scala:
-  - 2.11.12
+  - 2.12.12
 
-dist: trusty
+dist: precise
 
 jdk:
-  oraclejdk8
+  - oraclejdk8
+  - openjdk11
 
 stages:
   - name: test

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@
 </a>
 <!-- maven central -->
 <a href="https://mvnrepository.com/artifact/io.epiphanous/flinkrunner">
-  <img src="https://img.shields.io/maven-central/v/io.epiphanous/flinkrunner_2.11.svg" alt="maven" />
+  <img src="https://img.shields.io/maven-central/v/io.epiphanous/flinkrunner_2.12.svg" alt="maven" />
 </a>
 <!-- last commit -->
 <a href="https://github.com/epiphanous/flinkrunner/commits" title="Last Commit">
@@ -40,77 +40,26 @@
 
 ## Maven Dependency
 
-`Flinkrunner` `v2.0.1` is now on maven central, built against Flink 1.8.2 with Scala 2.11 and JDK 8.
+`Flinkrunner` `v3.0.0` is now on maven central, built against Flink 1.11.0 with Scala 2.12 and JDK 11.
 
 ```sbtshell
-libraryDependencies += "io.epiphanous" %% "flinkrunner" % "2.0.1"
+libraryDependencies += "io.epiphanous" %% "flinkrunner" % "2.1.0"
 ```
 
->The apache flink project doesn't include its AWS Kinesis connector on maven
-central because of license restrictions, and we don't include it in `FlinkRunner` for the same reasons.
-In order to use Kinesis with `FlinkRunner`, please follow the instructions in the next section.
-
-## Build from Source
-
-Due to licensing restrictions, if you want to use AWS Kinesis with `FlinkRunner` you have to build it (and the
-Flink kinesis connector) from source. To do so,
-
-* First, you'll need to build a local copy of Flink's kinesis connector. See
-  [these instructions](https://ci.apache.org/projects/flink/flink-docs-stable/dev/connectors/kinesis.html)
-  for details on how to accomplish that.
-
-  > Note that building Flink with Kinesis can take over 30 minutes! It's a very big project.
-
-* Clone the `FlinkRunner` repo:
-
-    ```bash
-    git clone https://github.com/epiphanous/flinkrunner
-    ```
-
-* Checkout the tag of `FlinkRunner` you want to build. The most recent stable version is
-  `v2.0.1`, but you can ensure you have the most recent tags with `git fetch --tags` and
-  list tags with `git tag -l`, then
-
-    ```bash
-    git checkout tags/v2.0.1 -b my-build-v2.0.1
-    ```
-
-   This will create a new local branch `my-build-v2.0.1` based on the `v2.0.1` tag release.
-
-* Build `FlinkRunner` and install it locally, using the `-Dwith.kinesis=true` option
-
-    ```bash
-    sbt -Dwith.kinesis=true publishLocal
-    ```
-
-  This will install `FlinkRunner` with kinesis support in your local repo.
-
-* In your project's build file, add a resolver to your local repo and add the local
-  `FlinkRunner` dependency:
-
-    ```sbtshell
-    resolvers += "Local Maven Repository" at "file://" +
-        Path.userHome.absolutePath + "/.m2/repository"
-    ...
-    libraryDependencies += "io.epiphanous" %% "flinkrunner" % "2.0.1k"
-                                      // notice no v here  ---^^    ^^---k for kinesis
-    ```
-
-
 ## What is FlinkRunner?
 
 You have a set of related flink jobs that deal in a related set of
 data event types. `Flinkrunner` helps you build one application to run
 those related jobs and coordinate the types. It also simplifies setting
 up common sources and sinks to the point where you control them purely
 with configuration and not code. `Flinkrunner` supports a variety of sources
-and sinks out of the box, including `kafka`, `kinesis`, `jdbc`, `filesystems`
-(including `s3`) and `sockets`. It also has many common
+and sinks out of the box, including `kafka`, `kinesis`, `jdbc`, `elasticsearch 7+` (sink only), 
+`cassandra` (sink only),  `filesystems` (including `s3`) and `sockets`. It also has many common
 operators to help you in writing your own transformation logic. Finally, `FlinkRunner`
 makes it easy to test your transformation logic with property-based testing.
 
-At a high level, `FlinkRunner` helps you think about your flink jobs at a high level,
-so you can focus on the logic in the pipeline between the sources and sinks.
+`FlinkRunner` helps you think about your flink jobs at a high level,
+so you can focus on the event pipeline, not the plumbing.
 
 ## Get Started
 

diff --git a/build.sbt b/build.sbt
@@ -1,67 +1,47 @@
 name := "flinkrunner"
 
-inThisBuild(List(
-  organization := "io.epiphanous",
-  homepage := Some(url("https://github.com/epiphanous/flinkrunner")),
-  licenses := List("MIT" -> url("https://opensource.org/licenses/MIT")),
-  developers := List(
-    Developer(
-      "nextdude",
-      "Robert Lyons",
-      "nextdude@gmail.com",
-      url("https://epiphanous.io")
-    )
-  )
-))
-
-ThisBuild / scalaVersion := "2.11.12"
+lazy val scala212 = "2.12.12"
+lazy val scala211 = "2.11.12"
+lazy val supportedScalaVersions = List(scala212, scala211)
+
+inThisBuild(
+  List(organization := "io.epiphanous",
+       homepage := Some(url("https://github.com/epiphanous/flinkrunner")),
+       licenses := List("MIT" -> url("https://opensource.org/licenses/MIT")),
+       developers := List(Developer("nextdude", "Robert Lyons", "nextdude@gmail.com", url("https://epiphanous.io"))),
+       scalaVersion := scala212)
+)
 
 Test / parallelExecution := false
-
 Test / fork := true
-
 resolvers += "Local Maven Repository" at "file://" + Path.userHome.absolutePath + "/.m2/repository"
 
 val V = new {
-  val flink = "1.8.2"
+  val flink = "1.11.0"
   val logback = "1.2.3"
-  val log4jOverSlf4j = "1.7.26"
+  val log4jOverSlf4j = "1.7.30"
   val scalaLogging = "3.9.2"
-  val scalaTest = "3.0.8"
-  val scalaCheck = "1.14.0"
-  val circe = "0.11.1"
-  val http4s = "0.20.10"
-  val enumeratum = "1.5.13"
+  val scalaTest = "3.2.0"
+  val scalaCheck = "1.14.3"
+  val circe = "0.13.0"
+  val http4s = "0.21.6"
+  val enumeratum = "1.6.1"
   val typesafeConfig = "1.3.4"
-  //  val guava = "27.0.1-jre"
-  val guava = "24.1-jre"
-  val squants = "1.3.0"
+  val guava = "29.0-jre"
+  val squants = "1.6.0"
+  val avro = "1.10.0"
+  val avro4s = "4.0.0-RC1"
+  val avro4s_211 = "3.0.0-RC2"
 }
 
-
-val withK = Seq("true","1","yes","y").exists(
-  _.equalsIgnoreCase(System.getProperty("with.kinesis", "false"))
-)
-
-val maybeKinesis = if (withK) Seq("connector-kinesis") else Seq.empty[String]
-
-// post-process version to add k suffix if we're building with kinesis
-val versionSuffix = if (withK) "k" else ""
-def suffixedVersion(v:String) = v + versionSuffix
-version in ThisBuild ~= suffixedVersion
-dynver in ThisBuild ~= suffixedVersion
-
 val flinkDeps = (
-  (Seq("scala", "streaming-scala", "cep-scala") ++ maybeKinesis).map(a =>
-    "org.apache.flink" %% s"flink-$a" % V.flink % Provided
-  ) ++
-  Seq("connector-kafka", "statebackend-rocksdb").map(a =>
-    "org.apache.flink" %% s"flink-$a" % V.flink
+  Seq("scala", "streaming-scala", "cep-scala").map(
+    a => "org.apache.flink" %% s"flink-$a" % V.flink % Provided
   ) ++
-  Seq(
-    "org.apache.flink" %% "flink-test-utils" % V.flink % Test
-  )
-  ).map(
+    Seq("connector-kafka", "connector-kinesis", "connector-cassandra",
+      "connector-elasticsearch7", "statebackend-rocksdb").map(a => "org.apache.flink" %% s"flink-$a" % V.flink) ++
+    Seq("org.apache.flink" %% "flink-test-utils" % V.flink % Test)
+).map(
   _.excludeAll(ExclusionRule(organization = "log4j"), ExclusionRule(organization = "org.slf4j", name = "slf4j-log4j12"))
 )
 
@@ -73,27 +53,38 @@ val loggingDeps = Seq("ch.qos.logback"             % "logback-core"     % V.logb
 val http4sDeps =
   Seq("http4s-dsl", "http4s-client", "http4s-blaze-client", "http4s-circe").map("org.http4s" %% _ % V.http4s)
 
-val otherDeps = Seq("com.beachape"      %% "enumeratum" % V.enumeratum,
-                    "com.typesafe"      %  "config"     % V.typesafeConfig,
-                    "com.google.guava"  %  "guava"      % V.guava,
-                    "org.typelevel"     %% "squants"    % V.squants,
-                    "org.scalactic"     %% "scalactic"  % V.scalaTest % Test,
-                    "org.scalatest"     %% "scalatest"  % V.scalaTest % Test,
-                    "org.scalacheck"    %% "scalacheck" % V.scalaCheck % Test
+val circeDeps = Seq("circe-core", "circe-generic", "circe-generic-extras", "circe-parser").map(
+  "io.circe" %% _ % V.circe
 )
 
-lazy val flink_runner =
-  (project in file(".")).settings(libraryDependencies ++= flinkDeps ++ loggingDeps ++ http4sDeps ++ otherDeps)
+val otherDeps = Seq("com.beachape"        %% "enumeratum"  % V.enumeratum,
+                    "org.apache.avro"     % "avro"         % V.avro,
+                    "com.typesafe"        % "config"       % V.typesafeConfig,
+                    "com.google.guava"    % "guava"        % V.guava,
+                    "org.typelevel"       %% "squants"     % V.squants,
+                    "org.scalactic"       %% "scalactic"   % V.scalaTest % Test,
+                    "org.scalatest"       %% "scalatest"   % V.scalaTest % Test,
+                    "org.scalacheck"      %% "scalacheck"  % V.scalaCheck % Test)
 
-scalacOptions ++= Seq(
-  "-encoding","utf8",
-  "-deprecation",
-  "-Xfuture",
-  "-Ywarn-dead-code",
-  "-Ywarn-numeric-widen",
-  "-Ywarn-unused",
-  "-Ywarn-value-discard"
-)
+lazy val flink_runner =
+  (project in file("."))
+    .settings(crossScalaVersions := supportedScalaVersions,
+              libraryDependencies ++=
+                flinkDeps ++ loggingDeps ++ http4sDeps ++ circeDeps ++ otherDeps ++ {
+                  CrossVersion.partialVersion(scalaVersion.value) match {
+                    case Some((2, scalaMinor)) if scalaMinor == 11 => Seq("com.sksamuel.avro4s" %% "avro4s-core" % V.avro4s_211)
+                    case _ => Seq("com.sksamuel.avro4s" %% "avro4s-core" % V.avro4s)
+                  }
+                })
+
+scalacOptions ++= Seq("-encoding",
+                      "utf8",
+                      "-deprecation",
+                      "-Xfuture",
+                      "-Ywarn-dead-code",
+                      "-Ywarn-numeric-widen",
+                      "-Ywarn-unused",
+                      "-Ywarn-value-discard")
 
 // stays inside the sbt console when we press "ctrl-c" while a Flink programme executes with "run" or "runMain"
 Compile / run / fork := true

diff --git a/kinesis.sbt b/kinesis.sbt
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.2.8
+sbt.version=1.3.13
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,3 +1,3 @@
-addSbtPlugin("com.geirsson"  % "sbt-ci-release" % "1.2.6")
-addSbtPlugin("org.scoverage" % "sbt-scoverage"  % "1.6.0")
+addSbtPlugin("com.geirsson"  % "sbt-ci-release" % "1.5.3")
+addSbtPlugin("org.scoverage" % "sbt-scoverage"  % "1.6.1")
 addSbtPlugin("org.scoverage" % "sbt-coveralls"  % "1.2.7")