Skip to content

Commit

Permalink
Create runners/core module for artifact org.apache.beam:runners-core
Browse files Browse the repository at this point in the history
This is strictly creating the module and moving one easy class to it.
Many of the utilities in org.apache.beam.util and subpackages should
move as developments allow.
  • Loading branch information
kennknowles committed May 4, 2016
1 parent 6819dff commit 892ead2
Show file tree
Hide file tree
Showing 9 changed files with 260 additions and 6 deletions.
6 changes: 6 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,12 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>runners-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>google-cloud-dataflow-java-runner</artifactId>
Expand Down
208 changes: 208 additions & 0 deletions runners/core-java/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.beam</groupId>
<artifactId>runners-parent</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>runners-core</artifactId>
<name>Apache Beam :: Runners :: Core</name>
<description>Beam Runners Core provides utilities to aid runner authors.</description>

<packaging>jar</packaging>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<goals><goal>analyze-only</goal></goals>
<configuration>
<failOnWarning>true</failOnWarning>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>

<!-- Source plugin for generating source and test-source JARs. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<windowtitle>Beam Runners Core utilities ${project.version} API</windowtitle>
<doctitle>Beam Runners Core utilities for Java, version ${project.version}</doctitle>
<overview>../javadoc/overview.html</overview>

<subpackages>org.apache.beam.runners.core</subpackages>
<use>false</use>
<quiet>true</quiet>
<bottom><![CDATA[<br>]]></bottom>
</configuration>
<executions>
<execution>
<goals>
<goal>jar</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<executions>
<!-- In the first phase, we pick dependencies and relocate them. -->
<execution>
<id>bundle-and-repackage</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadeTestJar>true</shadeTestJar>
<artifactSet>
<includes>
<include>com.google.guava:guava</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
<relocations>
<!-- TODO: Once ready, change the following pattern to 'com'
only, exclude 'org.apache.beam.**', and remove
the second relocation. -->
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.beam.sdk.repackaged.com.google.common</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.thirdparty</pattern>
<shadedPattern>org.apache.beam.sdk.repackaged.com.google.thirdparty</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>

<!-- In the second phase, we pick remaining dependencies and bundle
them without repackaging. -->
<execution>
<id>bundle-rest-without-repackaging</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadeTestJar>true</shadeTestJar>
<finalName>${project.artifactId}-bundled-${project.version}</finalName>
<artifactSet>
<excludes>
<exclude>com.google.guava:guava</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>

<!-- Coverage analysis for unit tests. -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>

</plugins>
</build>

<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>java-sdk-all</artifactId>
</dependency>

<!-- build dependencies -->

<!-- test dependencies -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.beam.sdk.util;
package org.apache.beam.runners.core;

import org.apache.beam.sdk.transforms.Aggregator;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.DoFnRunner.ReduceFnExecutor;
import org.apache.beam.sdk.util.GroupAlsoByWindowsDoFn;
import org.apache.beam.sdk.util.KeyedWorkItem;
import org.apache.beam.sdk.util.ReduceFnRunner;
import org.apache.beam.sdk.util.SystemDoFnInternal;
import org.apache.beam.sdk.util.SystemReduceFn;
import org.apache.beam.sdk.util.TimerInternals;
import org.apache.beam.sdk.util.TimerInternals.TimerData;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.util.state.StateInternals;
import org.apache.beam.sdk.values.KV;

Expand Down
Empty file.
21 changes: 21 additions & 0 deletions runners/direct-java/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,27 @@
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>java-sdk-all</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>runners-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<version>${google-clients.version}</version>
<exclusions>
<!-- Exclude an old version of guava that is being pulled
in by a transitive dependency of google-api-client -->
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import static org.apache.beam.sdk.util.CoderUtils.encodeToByteArray;

import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn;
import org.apache.beam.runners.direct.InProcessPipelineRunner.CommittedBundle;
import org.apache.beam.runners.direct.InProcessPipelineRunner.UncommittedBundle;
import org.apache.beam.runners.direct.StepTransformResult.Builder;
Expand All @@ -32,7 +33,6 @@
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.GroupAlsoByWindowViaWindowSetDoFn;
import org.apache.beam.sdk.util.GroupByKeyViaGroupByKeyOnly.ReifyTimestampsAndWindows;
import org.apache.beam.sdk.util.KeyedWorkItem;
import org.apache.beam.sdk.util.KeyedWorkItemCoder;
Expand Down
11 changes: 11 additions & 0 deletions runners/flink/runner/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,17 @@
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>runners-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>google-cloud-dataflow-java-runner</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/
package org.apache.beam.runners.flink.translation.wrappers.streaming;

import org.apache.beam.runners.core.GroupAlsoByWindowViaWindowSetDoFn;
import org.apache.beam.runners.flink.translation.types.CoderTypeInformation;
import org.apache.beam.runners.flink.translation.utils.SerializedPipelineOptions;
import org.apache.beam.runners.flink.translation.wrappers.SerializableFnAggregatorWrapper;
Expand All @@ -37,7 +38,6 @@
import org.apache.beam.sdk.transforms.windowing.OutputTimeFn;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.AppliedCombineFn;
import org.apache.beam.sdk.util.GroupAlsoByWindowViaWindowSetDoFn;
import org.apache.beam.sdk.util.KeyedWorkItem;
import org.apache.beam.sdk.util.KeyedWorkItems;
import org.apache.beam.sdk.util.SystemReduceFn;
Expand Down Expand Up @@ -83,7 +83,7 @@
* This class is the key class implementing all the windowing/triggering logic of Apache Beam.
* To provide full compatibility and support for all the windowing/triggering combinations offered by
* Beam, we opted for a strategy that uses the SDK's code for doing these operations. See the code in
* ({@link org.apache.beam.sdk.util.GroupAlsoByWindowsDoFn}.
* ({@link org.apache.beam.runners.core.GroupAlsoByWindowsDoFn}.
* <p/>
* In a nutshell, when the execution arrives to this operator, we expect to have a stream <b>already
* grouped by key</b>. Each of the elements that enter here, registers a timer
Expand All @@ -95,7 +95,7 @@
* When a watermark arrives, all the registered timers are checked to see which ones are ready to
* fire (see {@link FlinkGroupAlsoByWindowWrapper#processWatermark(Watermark)}). These are deregistered from
* the {@link FlinkGroupAlsoByWindowWrapper#activeTimers}
* list, and are fed into the {@link org.apache.beam.sdk.util.GroupAlsoByWindowsDoFn}
* list, and are fed into the {@link org.apache.beam.runners.core.GroupAlsoByWindowsDoFn}
* for furhter processing.
*/
public class FlinkGroupAlsoByWindowWrapper<K, VIN, VACC, VOUT>
Expand Down Expand Up @@ -253,7 +253,7 @@ public void open() throws Exception {
}

/**
* Create the adequate {@link org.apache.beam.sdk.util.GroupAlsoByWindowsDoFn},
* Create the adequate {@link org.apache.beam.runners.core.GroupAlsoByWindowsDoFn},
* <b> if not already created</b>.
* If a {@link org.apache.beam.sdk.transforms.Combine.KeyedCombineFn} was provided, then
* a function with that combiner is created, so that elements are combined as they arrive. This is
Expand Down
1 change: 1 addition & 0 deletions runners/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
<name>Apache Beam :: Runners</name>

<modules>
<module>core-java</module>
<module>direct-java</module>
<module>flink</module>
<module>spark</module>
Expand Down

0 comments on commit 892ead2

Please sign in to comment.