Skip to content

Commit 8427be3

Browse files
rayokotarayokota
committed
First commit of Kafka Streams Runner
1 parent a7226cb commit 8427be3

File tree

66 files changed

+7963
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+7963
-0
lines changed

‎pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -705,6 +705,12 @@
705705
<version>${project.version}</version>
706706
</dependency>
707707

708+
<dependency>
709+
<groupId>org.apache.beam</groupId>
710+
<artifactId>beam-runners-kafka-stream</artifactId>
711+
<version>${project.version}</version>
712+
</dependency>
713+
708714
<dependency>
709715
<groupId>org.apache.beam</groupId>
710716
<artifactId>beam-examples-java</artifactId>

‎runners/kafka-streams/pom.xml‎

Lines changed: 361 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,361 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
Unless required by applicable law or agreed to in writing,
12+
software distributed under the License is distributed on an
13+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
KIND, either express or implied. See the License for the
15+
specific language governing permissions and limitations
16+
under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
20+
<modelVersion>4.0.0</modelVersion>
21+
22+
<parent>
23+
<groupId>org.apache.beam</groupId>
24+
<artifactId>beam-runners-parent</artifactId>
25+
<version>2.4.0-SNAPSHOT</version>
26+
<relativePath>../pom.xml</relativePath>
27+
</parent>
28+
29+
<artifactId>beam-runners-kafka-streams</artifactId>
30+
31+
<name>Apache Beam :: Runners :: Kafka Streams</name>
32+
<packaging>jar</packaging>
33+
34+
<properties>
35+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
36+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
37+
<kafka.version>1.0.0</kafka.version>
38+
<confluent.version>4.0.0</confluent.version>
39+
</properties>
40+
41+
<profiles>
42+
<profile>
43+
<id>local-validates-runner-tests</id>
44+
<activation><activeByDefault>true</activeByDefault></activation>
45+
<build>
46+
<plugins>
47+
<plugin>
48+
<groupId>org.apache.maven.plugins</groupId>
49+
<artifactId>maven-surefire-plugin</artifactId>
50+
<executions>
51+
52+
<!-- This configures the inherited validates-runner-tests
53+
execution to execute with a local Kafka instance. -->
54+
<execution>
55+
<id>validates-runner-tests</id>
56+
<phase>integration-test</phase>
57+
<goals>
58+
<goal>test</goal>
59+
</goals>
60+
<configuration>
61+
<groups>org.apache.beam.sdk.testing.ValidatesRunner</groups>
62+
<excludedGroups>
63+
org.apache.beam.sdk.testing.LargeKeys$Above1MB,
64+
org.apache.beam.sdk.testing.UsesStatefulParDo,
65+
org.apache.beam.sdk.testing.UsesTimersInParDo,
66+
org.apache.beam.sdk.testing.UsesSplittableParDo,
67+
org.apache.beam.sdk.testing.UsesAttemptedMetrics,
68+
org.apache.beam.sdk.testing.UsesCommittedMetrics,
69+
org.apache.beam.sdk.testing.UsesTestStream
70+
</excludedGroups>
71+
<parallel>none</parallel>
72+
<failIfNoTests>true</failIfNoTests>
73+
<dependenciesToScan>
74+
<dependency>org.apache.beam:beam-sdks-java-core</dependency>
75+
</dependenciesToScan>
76+
<!--
77+
<includes>
78+
<include>
79+
org.apache.beam.sdk.PipelineTest
80+
</include>
81+
</includes>
82+
-->
83+
<systemPropertyVariables>
84+
<beamTestPipelineOptions>
85+
[
86+
"--runner=TestKafkaStreamsRunner"
87+
]
88+
</beamTestPipelineOptions>
89+
</systemPropertyVariables>
90+
<threadCount>4</threadCount>
91+
</configuration>
92+
</execution>
93+
</executions>
94+
</plugin>
95+
</plugins>
96+
</build>
97+
</profile>
98+
</profiles>
99+
100+
<dependencies>
101+
<dependency>
102+
<groupId>org.apache.kafka</groupId>
103+
<artifactId>kafka-streams</artifactId>
104+
<version>${kafka.version}</version>
105+
</dependency>
106+
<dependency>
107+
<groupId>org.apache.kafka</groupId>
108+
<artifactId>kafka-streams</artifactId>
109+
<version>${kafka.version}</version>
110+
<classifier>test</classifier>
111+
<scope>test</scope>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.kafka</groupId>
115+
<artifactId>kafka-clients</artifactId>
116+
<version>${kafka.version}</version>
117+
</dependency>
118+
<dependency>
119+
<groupId>org.apache.kafka</groupId>
120+
<artifactId>kafka-clients</artifactId>
121+
<version>${kafka.version}</version>
122+
<classifier>test</classifier>
123+
<scope>test</scope>
124+
</dependency>
125+
<dependency>
126+
<groupId>org.apache.kafka</groupId>
127+
<artifactId>kafka_2.11</artifactId>
128+
<version>${kafka.version}</version>
129+
<scope>test</scope>
130+
</dependency>
131+
<dependency>
132+
<groupId>org.apache.kafka</groupId>
133+
<artifactId>kafka_2.11</artifactId>
134+
<version>${kafka.version}</version>
135+
<classifier>test</classifier>
136+
<scope>test</scope>
137+
</dependency>
138+
<!-- Dependencies below are required/recommended only when using Apache Avro. -->
139+
<!--
140+
<dependency>
141+
<groupId>io.confluent</groupId>
142+
<artifactId>kafka-avro-serializer</artifactId>
143+
<version>${confluent.version}</version>
144+
</dependency>
145+
-->
146+
<!-- Beam -->
147+
<dependency>
148+
<groupId>org.apache.beam</groupId>
149+
<artifactId>beam-sdks-java-core</artifactId>
150+
<exclusions>
151+
<exclusion>
152+
<groupId>org.slf4j</groupId>
153+
<artifactId>slf4j-jdk14</artifactId>
154+
</exclusion>
155+
</exclusions>
156+
</dependency>
157+
158+
<dependency>
159+
<groupId>org.apache.beam</groupId>
160+
<artifactId>beam-runners-core-java</artifactId>
161+
<exclusions>
162+
<exclusion>
163+
<groupId>org.slf4j</groupId>
164+
<artifactId>slf4j-jdk14</artifactId>
165+
</exclusion>
166+
</exclusions>
167+
</dependency>
168+
169+
<dependency>
170+
<groupId>org.apache.beam</groupId>
171+
<artifactId>beam-runners-core-construction-java</artifactId>
172+
<exclusions>
173+
<exclusion>
174+
<groupId>org.slf4j</groupId>
175+
<artifactId>slf4j-jdk14</artifactId>
176+
</exclusion>
177+
</exclusions>
178+
</dependency>
179+
180+
<dependency>
181+
<groupId>org.slf4j</groupId>
182+
<artifactId>slf4j-api</artifactId>
183+
</dependency>
184+
<dependency>
185+
<groupId>joda-time</groupId>
186+
<artifactId>joda-time</artifactId>
187+
</dependency>
188+
<dependency>
189+
<groupId>com.google.guava</groupId>
190+
<artifactId>guava</artifactId>
191+
</dependency>
192+
<dependency>
193+
<groupId>com.google.auto.value</groupId>
194+
<artifactId>auto-value</artifactId>
195+
<version>1.5</version>
196+
<scope>provided</scope>
197+
</dependency>
198+
<dependency>
199+
<groupId>com.google.code.findbugs</groupId>
200+
<artifactId>jsr305</artifactId>
201+
</dependency>
202+
<dependency>
203+
<groupId>commons-io</groupId>
204+
<artifactId>commons-io</artifactId>
205+
<version>2.4</version>
206+
<scope>test</scope>
207+
</dependency>
208+
<dependency>
209+
<groupId>junit</groupId>
210+
<artifactId>junit</artifactId>
211+
<scope>test</scope>
212+
</dependency>
213+
<dependency>
214+
<groupId>org.hamcrest</groupId>
215+
<artifactId>hamcrest-all</artifactId>
216+
<scope>test</scope>
217+
</dependency>
218+
<dependency>
219+
<groupId>org.hamcrest</groupId>
220+
<artifactId>hamcrest-core</artifactId>
221+
<scope>test</scope>
222+
</dependency>
223+
<dependency>
224+
<groupId>com.fasterxml.jackson.core</groupId>
225+
<artifactId>jackson-databind</artifactId>
226+
<scope>test</scope>
227+
</dependency>
228+
<dependency>
229+
<groupId>org.apache.beam</groupId>
230+
<artifactId>beam-sdks-java-core</artifactId>
231+
<classifier>tests</classifier>
232+
<scope>test</scope>
233+
<exclusions>
234+
<exclusion>
235+
<groupId>org.slf4j</groupId>
236+
<artifactId>slf4j-jdk14</artifactId>
237+
</exclusion>
238+
</exclusions>
239+
</dependency>
240+
<dependency>
241+
<groupId>com.fasterxml.jackson.dataformat</groupId>
242+
<artifactId>jackson-dataformat-yaml</artifactId>
243+
<scope>test</scope>
244+
</dependency>
245+
<dependency>
246+
<groupId>org.mockito</groupId>
247+
<artifactId>mockito-all</artifactId>
248+
<scope>test</scope>
249+
</dependency>
250+
<dependency>
251+
<groupId>com.google.auto.service</groupId>
252+
<artifactId>auto-service</artifactId>
253+
</dependency>
254+
</dependencies>
255+
256+
<build>
257+
<plugins>
258+
<!-- JAR Packaging -->
259+
<plugin>
260+
<groupId>org.apache.maven.plugins</groupId>
261+
<artifactId>maven-jar-plugin</artifactId>
262+
<configuration>
263+
<archive>
264+
<manifest>
265+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
266+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
267+
</manifest>
268+
</archive>
269+
</configuration>
270+
</plugin>
271+
272+
<!-- Java compiler -->
273+
<plugin>
274+
<artifactId>maven-compiler-plugin</artifactId>
275+
<configuration>
276+
<source>1.8</source>
277+
<target>1.8</target>
278+
<testSource>1.8</testSource>
279+
<testTarget>1.8</testTarget>
280+
</configuration>
281+
</plugin>
282+
<plugin>
283+
<groupId>org.apache.maven.plugins</groupId>
284+
<artifactId>maven-shade-plugin</artifactId>
285+
<version>${maven-shade-plugin.version}</version>
286+
<executions>
287+
<execution>
288+
<id>bundle-and-repackage</id>
289+
<phase>package</phase>
290+
<goals>
291+
<goal>shade</goal>
292+
</goals>
293+
<configuration>
294+
<createDependencyReducedPom>false</createDependencyReducedPom>
295+
<shadeTestJar>true</shadeTestJar>
296+
<artifactSet>
297+
<includes>
298+
<include>com.google.guava:guava</include>
299+
</includes>
300+
</artifactSet>
301+
<filters>
302+
<filter>
303+
<artifact>*:*</artifact>
304+
<excludes>
305+
<exclude>META-INF/*.SF</exclude>
306+
<exclude>META-INF/*.DSA</exclude>
307+
<exclude>META-INF/*.RSA</exclude>
308+
</excludes>
309+
</filter>
310+
</filters>
311+
<relocations>
312+
<relocation>
313+
<pattern>com.google.common</pattern>
314+
<excludes>
315+
<!-- com.google.common is too generic, need to exclude guava-testlib -->
316+
<exclude>com.google.common.**.testing.*</exclude>
317+
</excludes>
318+
<!--suppress MavenModelInspection -->
319+
<shadedPattern>
320+
org.apache.${renderedArtifactId}.repackaged.com.google.common
321+
</shadedPattern>
322+
</relocation>
323+
<relocation>
324+
<pattern>com.google.thirdparty</pattern>
325+
<!--suppress MavenModelInspection -->
326+
<shadedPattern>
327+
org.apache.${renderedArtifactId}.repackaged.com.google.thirdparty
328+
</shadedPattern>
329+
</relocation>
330+
</relocations>
331+
<transformers>
332+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
333+
</transformers>
334+
</configuration>
335+
</execution>
336+
</executions>
337+
</plugin>
338+
339+
340+
<!-- uber jar -->
341+
<plugin>
342+
<groupId>org.apache.maven.plugins</groupId>
343+
<artifactId>maven-assembly-plugin</artifactId>
344+
<configuration>
345+
<descriptorRefs>
346+
<descriptorRef>jar-with-dependencies</descriptorRef>
347+
</descriptorRefs>
348+
</configuration>
349+
</plugin>
350+
351+
<plugin>
352+
<groupId>org.apache.maven.plugins</groupId>
353+
<artifactId>maven-checkstyle-plugin</artifactId>
354+
<configuration>
355+
<skip>true</skip>
356+
</configuration>
357+
</plugin>
358+
359+
</plugins>
360+
</build>
361+
</project>

0 commit comments

Comments
 (0)