From bfc1b447d8d93af69af98472bbe818ae61cca0d9 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 23 May 2017 21:40:35 +0200 Subject: [PATCH 1/5] [FLINK-6695] Activate strict checkstyle for flink-connector-wikiedits --- .../flink-connector-wikiedits/pom.xml | 39 +++++++++++++++++++ .../wikiedits/WikipediaEditEvent.java | 3 ++ .../wikiedits/WikipediaEditsSource.java | 6 ++- .../wikiedits/WikipediaEditsSourceTest.java | 6 ++- 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/flink-contrib/flink-connector-wikiedits/pom.xml b/flink-contrib/flink-connector-wikiedits/pom.xml index 20b6db498dff2c..1e5fbfdaf6fa83 100644 --- a/flink-contrib/flink-connector-wikiedits/pom.xml +++ b/flink-contrib/flink-connector-wikiedits/pom.xml @@ -47,4 +47,43 @@ under the License. 1.10 + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + /tools/maven/strict-checkstyle.xml + /tools/maven/suppressions.xml + true + true + true + + + + + test-compile + + check + + + + + + diff --git a/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditEvent.java b/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditEvent.java index ee3617d4816a54..a635c1e983fc1c 100644 --- a/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditEvent.java +++ b/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditEvent.java @@ -21,6 +21,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +/** + * Instances of this class represent edits made on Wikipedia. + */ public class WikipediaEditEvent { // Metadata diff --git a/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSource.java b/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSource.java index 943119c578f9bc..a10abdb1afab33 100644 --- a/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSource.java +++ b/flink-contrib/flink-connector-wikiedits/src/main/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSource.java @@ -23,6 +23,10 @@ import java.util.concurrent.TimeUnit; +/** + * This class is a SourceFunction that reads {@link WikipediaEditEvent} instances from the IRC channel + * #en.wikipedia. + */ public class WikipediaEditsSource extends RichSourceFunction { /** Hostname of the server to connect to. */ @@ -31,7 +35,7 @@ public class WikipediaEditsSource extends RichSourceFunction /** Port of the server to connect to. */ public static final int DEFAULT_PORT = 6667; - /** IRC channel to join */ + /** IRC channel to join. */ public static final String DEFAULT_CHANNEL = "#en.wikipedia"; private final String host; diff --git a/flink-contrib/flink-connector-wikiedits/src/test/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSourceTest.java b/flink-contrib/flink-connector-wikiedits/src/test/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSourceTest.java index c10c56a3a47957..439aa36418def9 100644 --- a/flink-contrib/flink-connector-wikiedits/src/test/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSourceTest.java +++ b/flink-contrib/flink-connector-wikiedits/src/test/java/org/apache/flink/streaming/connectors/wikiedits/WikipediaEditsSourceTest.java @@ -21,6 +21,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.SinkFunction; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +33,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; +/** + * Tests for the WikipediaEditsSource. + */ public class WikipediaEditsSourceTest { private static final Logger LOG = LoggerFactory.getLogger(WikipediaEditsSourceTest.class); @@ -39,7 +43,7 @@ public class WikipediaEditsSourceTest { /** * NOTE: if you are behind a firewall you may need to use a SOCKS Proxy for this test. * - * We first check the connection to the IRC server. If it fails, this test + *

We first check the connection to the IRC server. If it fails, this test * is effectively ignored. * * @see Socks Proxy From aecf5e8f67fddb84b7cb54062dfb89a7f1f7f496 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 23 May 2017 22:05:19 +0200 Subject: [PATCH 2/5] [FLINK-6695] Activate strict checkstyle for flink-statebackend-rocksDB --- .../flink-statebackend-rocksdb/pom.xml | 39 ++++++++++ .../streaming/state/AbstractRocksDBState.java | 16 ++-- .../streaming/state/OptionsFactory.java | 14 ++-- .../streaming/state/PredefinedOptions.java | 28 +++---- .../state/RocksDBAggregatingState.java | 12 +-- .../streaming/state/RocksDBFoldingState.java | 7 +- .../state/RocksDBKeyedStateBackend.java | 52 +++++++------ .../streaming/state/RocksDBListState.java | 6 +- .../streaming/state/RocksDBMapState.java | 75 +++++++++---------- .../streaming/state/RocksDBReducingState.java | 8 +- .../streaming/state/RocksDBStateBackend.java | 19 ++--- .../state/RocksDBStateBackendFactory.java | 6 +- .../streaming/state/RocksDBValueState.java | 5 +- .../streaming/state/RocksDBStateBackend.java | 8 +- .../state/RocksDBAggregatingStateTest.java | 5 +- .../state/RocksDBAsyncSnapshotTest.java | 14 +--- .../streaming/state/RocksDBInitResetTest.java | 2 +- .../streaming/state/RocksDBListStateTest.java | 4 +- .../state/RocksDBMergeIteratorTest.java | 4 + .../state/RocksDBReducingStateTest.java | 8 +- .../state/RocksDBStateBackendConfigTest.java | 7 +- .../state/RocksDBStateBackendFactoryTest.java | 3 + .../state/RocksDBStateBackendTest.java | 6 +- .../state/RocksDbMultiClassLoaderTest.java | 4 +- .../benchmark/RocksDBPerformanceTest.java | 16 ++-- 25 files changed, 205 insertions(+), 163 deletions(-) diff --git a/flink-contrib/flink-statebackend-rocksdb/pom.xml b/flink-contrib/flink-statebackend-rocksdb/pom.xml index 527ca185c2b0c1..f3d9da5d637550 100644 --- a/flink-contrib/flink-statebackend-rocksdb/pom.xml +++ b/flink-contrib/flink-statebackend-rocksdb/pom.xml @@ -92,4 +92,43 @@ under the License. test + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + /tools/maven/strict-checkstyle.xml + /tools/maven/suppressions.xml + true + true + true + + + + + test-compile + + check + + + + + + diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java index ba7fb28920cea7..c06183571db9cb 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java @@ -28,8 +28,8 @@ import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.runtime.query.netty.message.KvStateRequestSerializer; -import org.apache.flink.runtime.state.internal.InternalKvState; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; +import org.apache.flink.runtime.state.internal.InternalKvState; import org.apache.flink.util.Preconditions; import org.rocksdb.ColumnFamilyHandle; @@ -52,19 +52,19 @@ public abstract class AbstractRocksDBState, V> implements InternalKvState, State { - /** Serializer for the namespace */ + /** Serializer for the namespace. */ final TypeSerializer namespaceSerializer; - /** The current namespace, which the next value methods will refer to */ + /** The current namespace, which the next value methods will refer to. */ private N currentNamespace; - /** Backend that holds the actual RocksDB instance where we store state */ + /** Backend that holds the actual RocksDB instance where we store state. */ protected RocksDBKeyedStateBackend backend; - /** The column family of this particular instance of state */ + /** The column family of this particular instance of state. */ protected ColumnFamilyHandle columnFamily; - /** State descriptor from which to create this state instance */ + /** State descriptor from which to create this state instance. */ protected final SD stateDesc; /** @@ -110,7 +110,7 @@ public void clear() { writeCurrentKeyWithGroupAndNamespace(); byte[] key = keySerializationStream.toByteArray(); backend.db.remove(columnFamily, writeOptions, key); - } catch (IOException|RocksDBException e) { + } catch (IOException | RocksDBException e) { throw new RuntimeException("Error while removing entry from RocksDB", e); } } @@ -220,7 +220,7 @@ private static void writeVariableIntBytes( value >>>= 8; } while (value != 0); } - + protected Tuple3 readKeyWithGroupAndNamespace(ByteArrayInputStreamWithPos inputStream, DataInputView inputView) throws IOException { int keyGroup = readKeyGroup(inputView); K key = readKey(inputStream, inputView); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/OptionsFactory.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/OptionsFactory.java index 863c5daa7b14f3..34f7f622c5dc99 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/OptionsFactory.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/OptionsFactory.java @@ -25,18 +25,18 @@ * A factory for {@link DBOptions} to be passed to the {@link RocksDBStateBackend}. * Options have to be created lazily by this factory, because the {@code Options} * class is not serializable and holds pointers to native code. - * + * *

A typical pattern to use this OptionsFactory is as follows: - * + * *

Java 8:

*
{@code
  * rocksDbBackend.setOptions( (currentOptions) -> currentOptions.setMaxOpenFiles(1024) );
  * }
- * + * *

Java 7:

*
{@code
  * rocksDbBackend.setOptions(new OptionsFactory() {
- *     
+ *
  *     public Options setOptions(Options currentOptions) {
  *         return currentOptions.setMaxOpenFiles(1024);
  *     }
@@ -49,11 +49,11 @@ public interface OptionsFactory extends java.io.Serializable {
 	 * This method should set the additional options on top of the current options object.
 	 * The current options object may contain pre-defined options based on flags that have
 	 * been configured on the state backend.
-	 * 
+	 *
 	 * 

It is important to set the options on the current object and return the result from * the setter methods, otherwise the pre-defined options may get lost. - * - * @param currentOptions The options object with the pre-defined options. + * + * @param currentOptions The options object with the pre-defined options. * @return The options object on which the additional options are set. */ DBOptions createDBOptions(DBOptions currentOptions); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.java index 93aac8564637ea..f606131a06f3ab 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/PredefinedOptions.java @@ -25,10 +25,10 @@ import org.rocksdb.StringAppendOperator; /** - * The {@code PredefinedOptions} are configuration settings for the {@link RocksDBStateBackend}. + * The {@code PredefinedOptions} are configuration settings for the {@link RocksDBStateBackend}. * The various pre-defined choices are configurations that have been empirically * determined to be beneficial for performance under different settings. - * + * *

Some of these settings are based on experiments by the Flink community, some follow * guides from the RocksDB project. */ @@ -37,12 +37,12 @@ public enum PredefinedOptions { /** * Default options for all settings, except that writes are not forced to the * disk. - * + * *

Note: Because Flink does not rely on RocksDB data on disk for recovery, * there is no need to sync data to stable storage. */ DEFAULT { - + @Override public DBOptions createDBOptions() { return new DBOptions() @@ -60,11 +60,11 @@ public ColumnFamilyOptions createColumnOptions() { /** * Pre-defined options for regular spinning hard disks. - * + * *

This constant configures RocksDB with some options that lead empirically * to better performance when the machines executing the system use * regular spinning hard disks. - * + * *

The following options are set: *

    *
  • setCompactionStyle(CompactionStyle.LEVEL)
  • @@ -74,7 +74,7 @@ public ColumnFamilyOptions createColumnOptions() { *
  • setDisableDataSync(true)
  • *
  • setMaxOpenFiles(-1)
  • *
- * + * *

Note: Because Flink does not rely on RocksDB data on disk for recovery, * there is no need to sync data to stable storage. */ @@ -121,7 +121,7 @@ public ColumnFamilyOptions createColumnOptions() { *

  • BlockBasedTableConfig.setBlockCacheSize(256 MBytes)
  • *
  • BlockBasedTableConfigsetBlockSize(128 KBytes)
  • * - * + * *

    Note: Because Flink does not rely on RocksDB data on disk for recovery, * there is no need to sync data to stable storage. */ @@ -161,13 +161,13 @@ public ColumnFamilyOptions createColumnOptions() { ); } }, - + /** * Pre-defined options for Flash SSDs. * *

    This constant configures RocksDB with some options that lead empirically * to better performance when the machines executing the system use SSDs. - * + * *

    The following options are set: *

      *
    • setIncreaseParallelism(4)
    • @@ -175,7 +175,7 @@ public ColumnFamilyOptions createColumnOptions() { *
    • setDisableDataSync(true)
    • *
    • setMaxOpenFiles(-1)
    • *
    - * + * *

    Note: Because Flink does not rely on RocksDB data on disk for recovery, * there is no need to sync data to stable storage. */ @@ -196,13 +196,13 @@ public ColumnFamilyOptions createColumnOptions() { .setMergeOperator(new StringAppendOperator()); } }; - + // ------------------------------------------------------------------------ /** * Creates the {@link DBOptions}for this pre-defined setting. - * - * @return The pre-defined options object. + * + * @return The pre-defined options object. */ public abstract DBOptions createDBOptions(); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java index 1f306b44e1cda2..fc84456f2a0149 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java @@ -25,8 +25,8 @@ import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; - import org.apache.flink.runtime.state.internal.InternalAggregatingState; + import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.rocksdb.WriteOptions; @@ -47,10 +47,10 @@ public class RocksDBAggregatingState extends AbstractRocksDBState, AggregatingStateDescriptor, ACC> implements InternalAggregatingState { - /** Serializer for the values */ + /** Serializer for the values. */ private final TypeSerializer valueSerializer; - /** User-specified aggregation function */ + /** User-specified aggregation function. */ private final AggregateFunction aggFunction; /** @@ -64,7 +64,7 @@ public class RocksDBAggregatingState * * @param namespaceSerializer * The serializer for the namespace. - * @param stateDesc + * @param stateDesc * The state identifier for the state. This contains the state name and aggregation function. */ public RocksDBAggregatingState( @@ -154,7 +154,7 @@ public void mergeNamespaces(N target, Collection sources) throws Exception { writeKeyWithGroupAndNamespace( keyGroup, key, source, keySerializationStream, keySerializationDataOutputView); - + final byte[] sourceKey = keySerializationStream.toByteArray(); final byte[] valueBytes = backend.db.get(columnFamily, sourceKey); @@ -174,7 +174,7 @@ public void mergeNamespaces(N target, Collection sources) throws Exception { // if something came out of merging the sources, merge it or write it to the target if (current != null) { - // create the target full-binary-key + // create the target full-binary-key writeKeyWithGroupAndNamespace( keyGroup, key, target, keySerializationStream, keySerializationDataOutputView); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java index d5d9fce0189b30..479565e359e484 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java @@ -26,6 +26,7 @@ import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.runtime.state.internal.InternalFoldingState; + import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.rocksdb.WriteOptions; @@ -47,10 +48,10 @@ public class RocksDBFoldingState extends AbstractRocksDBState, FoldingStateDescriptor, ACC> implements InternalFoldingState { - /** Serializer for the values */ + /** Serializer for the values. */ private final TypeSerializer valueSerializer; - /** User-specified fold function */ + /** User-specified fold function. */ private final FoldFunction foldFunction; /** @@ -90,7 +91,7 @@ public ACC get() { return null; } return valueSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(valueBytes))); - } catch (IOException|RocksDBException e) { + } catch (IOException | RocksDBException e) { throw new RuntimeException("Error while retrieving data from RocksDB", e); } } diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 053c820b999d6b..241c0b362b1f98 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -26,6 +26,7 @@ import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.CompatibilityResult; +import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer; @@ -53,7 +54,6 @@ import org.apache.flink.runtime.query.TaskKvStateRegistry; import org.apache.flink.runtime.state.AbstractKeyedStateBackend; import org.apache.flink.runtime.state.CheckpointStreamFactory; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.runtime.state.DoneFuture; import org.apache.flink.runtime.state.IncrementalKeyedStateHandle; import org.apache.flink.runtime.state.KeyGroupRange; @@ -79,6 +79,7 @@ import org.apache.flink.util.IOUtils; import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.Preconditions; + import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; @@ -125,16 +126,16 @@ public class RocksDBKeyedStateBackend extends AbstractKeyedStateBackend { private final String operatorIdentifier; - /** The column family options from the options factory */ + /** The column family options from the options factory. */ private final ColumnFamilyOptions columnOptions; - /** The DB options from the options factory */ + /** The DB options from the options factory. */ private final DBOptions dbOptions; - /** Path where this configured instance stores its data directory */ + /** Path where this configured instance stores its data directory. */ private final File instanceBasePath; - /** Path where this configured instance stores its RocksDB data base */ + /** Path where this configured instance stores its RocksDB data base. */ private final File instanceRocksDBPath; /** @@ -160,7 +161,7 @@ public class RocksDBKeyedStateBackend extends AbstractKeyedStateBackend { /** * Map of state names to their corresponding restored state meta info. * - * TODO this map can be removed when eager-state registration is in place. + *

    TODO this map can be removed when eager-state registration is in place. * TODO we currently need this cached to check state migration strategies when new serializers are registered. */ private final Map> restoredKvStateMetaInfos; @@ -168,13 +169,13 @@ public class RocksDBKeyedStateBackend extends AbstractKeyedStateBackend { /** Number of bytes required to prefix the key groups. */ private final int keyGroupPrefixBytes; - /** True if incremental checkpointing is enabled */ + /** True if incremental checkpointing is enabled. */ private final boolean enableIncrementalCheckpointing; - /** The state handle ids of all sst files materialized in snapshots for previous checkpoints */ + /** The state handle ids of all sst files materialized in snapshots for previous checkpoints. */ private final SortedMap> materializedSstFiles; - /** The identifier of the last completed checkpoint */ + /** The identifier of the last completed checkpoint. */ private long lastCompletedCheckpointId = -1; private static final String SST_FILE_SUFFIX = ".sst"; @@ -711,22 +712,22 @@ private static void checkInterrupted() throws InterruptedException { private static final class RocksDBIncrementalSnapshotOperation { - /** The backend which we snapshot */ + /** The backend which we snapshot. */ private final RocksDBKeyedStateBackend stateBackend; - /** Stream factory that creates the outpus streams to DFS */ + /** Stream factory that creates the outpus streams to DFS. */ private final CheckpointStreamFactory checkpointStreamFactory; - /** Id for the current checkpoint */ + /** Id for the current checkpoint. */ private final long checkpointId; - /** Timestamp for the current checkpoint */ + /** Timestamp for the current checkpoint. */ private final long checkpointTimestamp; - /** All sst files that were part of the last previously completed checkpoint */ + /** All sst files that were part of the last previously completed checkpoint. */ private Set baseSstFiles; - /** The state meta data */ + /** The state meta data. */ private final List> stateMetaInfoSnapshots = new ArrayList<>(); private FileSystem backupFileSystem; @@ -888,8 +889,6 @@ KeyedStateHandle materializeSnapshot() throws Exception { } } - - synchronized (stateBackend.materializedSstFiles) { stateBackend.materializedSstFiles.put(checkpointId, sstFiles.keySet()); } @@ -1036,13 +1035,13 @@ static final class RocksDBFullRestoreOperation { private final RocksDBKeyedStateBackend rocksDBKeyedStateBackend; - /** Current key-groups state handle from which we restore key-groups */ + /** Current key-groups state handle from which we restore key-groups. */ private KeyGroupsStateHandle currentKeyGroupsStateHandle; - /** Current input stream we obtained from currentKeyGroupsStateHandle */ + /** Current input stream we obtained from currentKeyGroupsStateHandle. */ private FSDataInputStream currentStateHandleInStream; - /** Current data input view that wraps currentStateHandleInStream */ + /** Current data input view that wraps currentStateHandleInStream. */ private DataInputView currentStateHandleInView; - /** Current list of ColumnFamilyHandles for all column families we restore from currentKeyGroupsStateHandle */ + /** Current list of ColumnFamilyHandles for all column families we restore from currentKeyGroupsStateHandle. */ private List currentStateHandleKVStateColumnFamilies; /** @@ -1082,7 +1081,7 @@ public void doRestore(Collection keyedStateHandles) } /** - * Restore one key groups state handle + * Restore one key groups state handle. * * @throws IOException * @throws RocksDBException @@ -1105,7 +1104,7 @@ private void restoreKeyGroupsInStateHandle() } /** - * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle + * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle. * * @throws IOException * @throws ClassNotFoundException @@ -1169,7 +1168,7 @@ private void restoreKVStateMetaData() throws IOException, RocksDBException { } /** - * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state handle + * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state handle. * * @throws IOException * @throws RocksDBException @@ -1376,7 +1375,7 @@ private void restoreInstance( int startKeyGroup = stateBackend.getKeyGroupRange().getStartKeyGroup(); byte[] startKeyGroupPrefixBytes = new byte[stateBackend.keyGroupPrefixBytes]; for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) { - startKeyGroupPrefixBytes[j] = (byte)(startKeyGroup >>> ((stateBackend.keyGroupPrefixBytes - j - 1) * Byte.SIZE)); + startKeyGroupPrefixBytes[j] = (byte) (startKeyGroup >>> ((stateBackend.keyGroupPrefixBytes - j - 1) * Byte.SIZE)); } iterator.seek(startKeyGroupPrefixBytes); @@ -1430,7 +1429,6 @@ private void restoreInstance( columnFamilyHandle, stateMetaInfo)); } - // use the restore sst files as the base for succeeding checkpoints synchronized (stateBackend.materializedSstFiles) { stateBackend.materializedSstFiles.put(restoreStateHandle.getCheckpointId(), sstFiles.keySet()); @@ -1480,7 +1478,7 @@ void restore(Collection restoreStateHandles) throws Exception for (KeyedStateHandle rawStateHandle : restoreStateHandles) { - if (! (rawStateHandle instanceof IncrementalKeyedStateHandle)) { + if (!(rawStateHandle instanceof IncrementalKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java index a8b20d19c387c4..9d3e97ef35cc58 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java @@ -50,7 +50,7 @@ public class RocksDBListState extends AbstractRocksDBState, ListStateDescriptor, List> implements InternalListState { - /** Serializer for the values */ + /** Serializer for the values. */ private final TypeSerializer valueSerializer; /** @@ -100,7 +100,7 @@ public Iterable get() { } } return result; - } catch (IOException|RocksDBException e) { + } catch (IOException | RocksDBException e) { throw new RuntimeException("Error while retrieving data from RocksDB", e); } } @@ -131,7 +131,7 @@ public void mergeNamespaces(N target, Collection sources) throws Exception { final int keyGroup = backend.getCurrentKeyGroupIndex(); try { - // create the target full-binary-key + // create the target full-binary-key writeKeyWithGroupAndNamespace( keyGroup, key, target, keySerializationStream, keySerializationDataOutputView); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java index 5125240f96c1a3..75c165187e288b 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java @@ -30,6 +30,7 @@ import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.runtime.state.internal.InternalMapState; import org.apache.flink.util.Preconditions; + import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; @@ -45,7 +46,7 @@ /** * {@link MapState} implementation that stores state in RocksDB. - *

    + * *

    {@link RocksDBStateBackend} must ensure that we set the * {@link org.rocksdb.StringAppendOperator} on the column family that we use for our state since * we use the {@code merge()} call. @@ -58,10 +59,10 @@ public class RocksDBMapState extends AbstractRocksDBState, MapStateDescriptor, Map> implements InternalMapState { - - private static Logger LOG = LoggerFactory.getLogger(RocksDBMapState.class); - /** Serializer for the keys and values */ + private static final Logger LOG = LoggerFactory.getLogger(RocksDBMapState.class); + + /** Serializer for the keys and values. */ private final TypeSerializer userKeySerializer; private final TypeSerializer userValueSerializer; @@ -105,19 +106,19 @@ public UV get(UK userKey) throws IOException, RocksDBException { @Override public void put(UK userKey, UV userValue) throws IOException, RocksDBException { - + byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(userKey); byte[] rawValueBytes = serializeUserValue(userValue); backend.db.put(columnFamily, writeOptions, rawKeyBytes, rawValueBytes); } - + @Override public void putAll(Map map) throws IOException, RocksDBException { if (map == null) { return; } - + for (Map.Entry entry : map.entrySet()) { put(entry.getKey(), entry.getValue()); } @@ -137,7 +138,7 @@ public boolean contains(UK userKey) throws IOException, RocksDBException { return (rawValueBytes != null); } - + @Override public Iterable> entries() throws IOException, RocksDBException { final Iterator> iterator = iterator(); @@ -158,7 +159,7 @@ public Iterator> iterator() { @Override public Iterable keys() throws IOException, RocksDBException { final byte[] prefixBytes = serializeCurrentKeyAndNamespace(); - + return new Iterable() { @Override public Iterator iterator() { @@ -176,7 +177,7 @@ public UK next() { @Override public Iterable values() throws IOException, RocksDBException { final byte[] prefixBytes = serializeCurrentKeyAndNamespace(); - + return new Iterable() { @Override public Iterator iterator() { @@ -202,7 +203,7 @@ public Map.Entry next() { } }; } - + @Override public void clear() { try { @@ -216,7 +217,7 @@ public void clear() { LOG.warn("Error while cleaning the state.", e); } } - + @Override @SuppressWarnings("unchecked") public byte[] getSerializedValue(byte[] serializedKeyAndNamespace) throws Exception { @@ -229,7 +230,7 @@ public byte[] getSerializedValue(byte[] serializedKeyAndNamespace) throws Except namespaceSerializer); int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(des.f0, backend.getNumberOfKeyGroups()); - + ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(128); DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream); writeKeyWithGroupAndNamespace(keyGroup, des.f0, des.f1, outputStream, outputView); @@ -246,7 +247,7 @@ public Map.Entry next() { if (!iterator.hasNext()) { return null; } - + return KvStateRequestSerializer.serializeMap(new Iterable>() { @Override public Iterator> iterator() { @@ -254,21 +255,21 @@ public Iterator> iterator() { } }, userKeySerializer, userValueSerializer); } - + // ------------------------------------------------------------------------ // Serialization Methods // ------------------------------------------------------------------------ - + private byte[] serializeCurrentKeyAndNamespace() throws IOException { writeCurrentKeyWithGroupAndNamespace(); - + return keySerializationStream.toByteArray(); } private byte[] serializeUserKeyWithCurrentKeyAndNamespace(UK userKey) throws IOException { writeCurrentKeyWithGroupAndNamespace(); userKeySerializer.serialize(userKey, keySerializationDataOutputView); - + return keySerializationStream.toByteArray(); } @@ -282,7 +283,6 @@ private byte[] serializeUserValue(UV userValue) throws IOException { userValueSerializer.serialize(userValue, keySerializationDataOutputView); } - return keySerializationStream.toByteArray(); } @@ -291,7 +291,7 @@ private UK deserializeUserKey(byte[] rawKeyBytes) throws IOException { DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); readKeyWithGroupAndNamespace(bais, in); - + return userKeySerializer.deserialize(in); } @@ -303,20 +303,20 @@ private UV deserializeUserValue(byte[] rawValueBytes) throws IOException { return isNull ? null : userValueSerializer.deserialize(in); } - + // ------------------------------------------------------------------------ // Internal Classes // ------------------------------------------------------------------------ - - /** A map entry in RocksDBMapState */ + + /** A map entry in RocksDBMapState. */ private class RocksDBMapEntry implements Map.Entry { private final RocksDB db; - + /** The raw bytes of the key stored in RocksDB. Each user key is stored in RocksDB * with the format #KeyGroup#Key#Namespace#UserKey. */ private final byte[] rawKeyBytes; - - /** The raw bytes of the value stored in RocksDB */ + + /** The raw bytes of the value stored in RocksDB. */ private byte[] rawValueBytes; /** True if the entry has been deleted. */ @@ -329,7 +329,7 @@ private class RocksDBMapEntry implements Map.Entry { RocksDBMapEntry(final RocksDB db, final byte[] rawKeyBytes, final byte[] rawValueBytes) { this.db = db; - + this.rawKeyBytes = rawKeyBytes; this.rawValueBytes = rawValueBytes; this.deleted = false; @@ -383,7 +383,7 @@ public UV setValue(UV value) { } UV oldValue = getValue(); - + try { userValue = value; rawValueBytes = serializeUserValue(value); @@ -400,22 +400,22 @@ public UV setValue(UV value) { /** An auxiliary utility to scan all entries under the given key. */ private abstract class RocksDBMapIterator implements Iterator { - final static int CACHE_SIZE_BASE = 1; - final static int CACHE_SIZE_LIMIT = 128; + static final int CACHE_SIZE_BASE = 1; + static final int CACHE_SIZE_LIMIT = 128; /** The db where data resides. */ private final RocksDB db; - /** + /** * The prefix bytes of the key being accessed. All entries under the same key * has the same prefix, hence we can stop the iterating once coming across an - * entry with a different prefix. + * entry with a different prefix. */ private final byte[] keyPrefixBytes; /** * True if all entries have been accessed or the iterator has come across an - * entry with a different prefix. + * entry with a different prefix. */ private boolean expired = false; @@ -423,7 +423,6 @@ private abstract class RocksDBMapIterator implements Iterator { private ArrayList cacheEntries = new ArrayList<>(); private int cacheIndex = 0; - RocksDBMapIterator(final RocksDB db, final byte[] keyPrefixBytes) { this.db = db; this.keyPrefixBytes = keyPrefixBytes; @@ -440,7 +439,7 @@ public boolean hasNext() { public void remove() { if (cacheIndex == 0 || cacheIndex > cacheEntries.size()) { throw new IllegalStateException("The remove operation must be called after an valid next operation."); - } + } RocksDBMapEntry lastEntry = cacheEntries.get(cacheIndex - 1); lastEntry.remove(); @@ -489,7 +488,7 @@ private void loadCache() { iterator.seek(startBytes); - /* + /* * If the last returned entry is not deleted, it will be the first entry in the * iterating. Skip it to avoid redundant access in such cases. */ @@ -515,7 +514,7 @@ private void loadCache() { iterator.close(); } - + private boolean underSameKey(byte[] rawKeyBytes) { if (rawKeyBytes.length < keyPrefixBytes.length) { return false; @@ -530,4 +529,4 @@ private boolean underSameKey(byte[] rawKeyBytes) { return true; } } -} \ No newline at end of file +} diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java index ccc98a701acb56..b5fe95f5541c0d 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java @@ -46,10 +46,10 @@ public class RocksDBReducingState extends AbstractRocksDBState, ReducingStateDescriptor, V> implements InternalReducingState { - /** Serializer for the values */ + /** Serializer for the values. */ private final TypeSerializer valueSerializer; - /** User-specified reduce function */ + /** User-specified reduce function. */ private final ReduceFunction reduceFunction; /** @@ -88,7 +88,7 @@ public V get() { return null; } return valueSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStream(valueBytes))); - } catch (IOException|RocksDBException e) { + } catch (IOException | RocksDBException e) { throw new RuntimeException("Error while retrieving data from RocksDB", e); } } @@ -157,7 +157,7 @@ public void mergeNamespaces(N target, Collection sources) throws Exception { // if something came out of merging the sources, merge it or write it to the target if (current != null) { - // create the target full-binary-key + // create the target full-binary-key writeKeyWithGroupAndNamespace( keyGroup, key, target, keySerializationStream, keySerializationDataOutputView); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java index 2b70dcd0552af3..4a30489416a0fe 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java @@ -31,6 +31,7 @@ import org.apache.flink.runtime.state.OperatorStateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.util.AbstractID; + import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.DBOptions; import org.rocksdb.NativeLibraryLoader; @@ -69,10 +70,10 @@ public class RocksDBStateBackend extends AbstractStateBackend { private static final Logger LOG = LoggerFactory.getLogger(RocksDBStateBackend.class); - /** The number of (re)tries for loading the RocksDB JNI library */ + /** The number of (re)tries for loading the RocksDB JNI library. */ private static final int ROCKSDB_LIB_LOADING_ATTEMPTS = 3; - + private static boolean rocksDbInitialized = false; // ------------------------------------------------------------------------ @@ -93,23 +94,23 @@ public class RocksDBStateBackend extends AbstractStateBackend { /** Base paths for RocksDB directory, as configured. May be null. */ private Path[] configuredDbBasePaths; - /** Base paths for RocksDB directory, as initialized */ + /** Base paths for RocksDB directory, as initialized. */ private File[] initializedDbBasePaths; private int nextDirectory; // RocksDB options - /** The pre-configured option settings */ + /** The pre-configured option settings. */ private PredefinedOptions predefinedOptions = PredefinedOptions.DEFAULT; - /** The options factory to create the RocksDB options in the cluster */ + /** The options factory to create the RocksDB options in the cluster. */ private OptionsFactory optionsFactory; /** Whether we already lazily initialized our local storage directories. */ private transient boolean isInitialized = false; - /** True if incremental checkpointing is enabled */ + /** True if incremental checkpointing is enabled. */ private boolean enableIncrementalCheckpointing; @@ -183,10 +184,10 @@ public RocksDBStateBackend(URI checkpointDataUri, boolean enableIncrementalCheck * Creates a new {@code RocksDBStateBackend} that uses the given state backend to store its * checkpoint data streams. Typically, one would supply a filesystem or database state backend * here where the snapshots from RocksDB would be stored. - * + * *

    The snapshots of the RocksDB state will be stored using the given backend's - * {@link AbstractStateBackend#createStreamFactory(JobID, String) checkpoint stream}. - * + * {@link AbstractStateBackend#createStreamFactory(JobID, String) checkpoint stream}. + * * @param checkpointStreamBackend The backend to store the */ public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) { diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactory.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactory.java index bd9bcaa241be9b..f0569b8d61e936 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactory.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactory.java @@ -39,13 +39,13 @@ public class RocksDBStateBackendFactory implements StateBackendFactory extends AbstractRocksDBState, ValueStateDescriptor, V> implements InternalValueState { - /** Serializer for the values */ + /** Serializer for the values. */ private final TypeSerializer valueSerializer; /** @@ -80,7 +81,7 @@ public V value() { return stateDesc.getDefaultValue(); } return valueSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStream(valueBytes))); - } catch (IOException|RocksDBException e) { + } catch (IOException | RocksDBException e) { throw new RuntimeException("Error while retrieving data from RocksDB.", e); } } diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java index 695aa12b4221be..024d12ea1be97a 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/migration/contrib/streaming/state/RocksDBStateBackend.java @@ -68,6 +68,12 @@ public void close() throws IOException { } } + /** + * This class exists to provide a good error message if a user attempts to restore from a semi async snapshot. + * + *

    see FLINK-5468 + */ + @Deprecated public static class FinalSemiAsyncSnapshot { static { @@ -75,7 +81,7 @@ public static class FinalSemiAsyncSnapshot { } private static void throwExceptionOnLoadingThisClass() { - throw new RuntimeException("Attempt to requiresMigration RocksDB state created with semi async snapshot mode failed. " + throw new RuntimeException("Attempt to migrate RocksDB state created with semi async snapshot mode failed. " + "Unfortunately, this is not supported. Please create a new savepoint for the job using fully " + "async mode in Flink 1.1 and run migration again with the new savepoint."); } diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java index 1b6546612dbc7f..f3065abbe8ded4 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingStateTest.java @@ -36,8 +36,9 @@ import org.junit.rules.TemporaryFolder; import static java.util.Arrays.asList; -import static org.mockito.Mockito.*; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; /** * Tests for the {@link InternalAggregatingState} implementation on top of RocksDB. diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java index 812babbb46c6c3..d2edf0efaf768d 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBAsyncSnapshotTest.java @@ -19,7 +19,6 @@ package org.apache.flink.contrib.streaming.state; import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; @@ -61,12 +60,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; - import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; - import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -408,7 +405,7 @@ public void write(int b) throws IOException { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - if(closed) { + if (closed) { throw new IOException("Stream closed."); } super.write(b); @@ -422,7 +419,7 @@ public void write(byte[] b, int off, int len) throws IOException { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - if(closed) { + if (closed) { throw new IOException("Stream closed."); } super.write(b, off, len); @@ -439,7 +436,7 @@ public void close() { } } - public static class AsyncCheckpointOperator + private static class AsyncCheckpointOperator extends AbstractStreamOperator implements OneInputStreamOperator, StreamCheckpointedOperator { @@ -480,9 +477,4 @@ public void restoreState(FSDataInputStream in) throws Exception { } } - - public static class DummyMapFunction implements MapFunction { - @Override - public T map(T value) { return value; } - } } diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBInitResetTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBInitResetTest.java index 7343b56511dfb9..565f27d5feed18 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBInitResetTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBInitResetTest.java @@ -21,7 +21,7 @@ import org.junit.Test; /** - * This test checks that the RocksDB native code loader still responds to resetting the + * This test checks that the RocksDB native code loader still responds to resetting the init flag. */ public class RocksDBInitResetTest { diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java index e7efcfaf076bbe..c6ccd5dec7d599 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBListStateTest.java @@ -64,9 +64,9 @@ public void testAddAndGet() throws Exception { backend.setDbStoragePath(tmp.newFolder().getAbsolutePath()); final RocksDBKeyedStateBackend keyedBackend = createKeyedBackend(backend); - + try { - InternalListState state = + InternalListState state = keyedBackend.createListState(VoidNamespaceSerializer.INSTANCE, stateDescr); state.setCurrentNamespace(VoidNamespace.INSTANCE); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java index f5bcf8677bc0b3..1d14f6e92fe72e 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; + import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -38,6 +39,9 @@ import java.util.List; import java.util.Random; +/** + * Tests for the RocksDBMergeIterator. + */ public class RocksDBMergeIteratorTest { private static final int NUM_KEY_VAL_STATES = 50; diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java index a8b45353637a39..0733dce32e05b5 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBReducingStateTest.java @@ -54,7 +54,7 @@ public class RocksDBReducingStateTest { @Test public void testAddAndGet() throws Exception { - final ReducingStateDescriptor stateDescr = + final ReducingStateDescriptor stateDescr = new ReducingStateDescriptor<>("my-state", new AddingFunction(), Long.class); stateDescr.initializeSerializerUnlessSet(new ExecutionConfig()); @@ -62,9 +62,9 @@ public void testAddAndGet() throws Exception { backend.setDbStoragePath(tmp.newFolder().getAbsolutePath()); final RocksDBKeyedStateBackend keyedBackend = createKeyedBackend(backend); - + try { - InternalReducingState state = + InternalReducingState state = keyedBackend.createReducingState(VoidNamespaceSerializer.INSTANCE, stateDescr); state.setCurrentNamespace(VoidNamespace.INSTANCE); @@ -126,7 +126,7 @@ public void testMerging() throws Exception { final RocksDBKeyedStateBackend keyedBackend = createKeyedBackend(backend); try { - final InternalReducingState state = + final InternalReducingState state = keyedBackend.createReducingState(new TimeWindow.Serializer(), stateDescr); // populate the different namespaces diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java index 463dd440b892b7..ff433adc966d40 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendConfigTest.java @@ -18,7 +18,6 @@ package org.apache.flink.contrib.streaming.state; -import org.apache.commons.io.FileUtils; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.typeutils.base.IntSerializer; @@ -32,6 +31,8 @@ import org.apache.flink.runtime.state.KeyGroupRange; import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo; import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo; + +import org.apache.commons.io.FileUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -59,7 +60,7 @@ /** - * Tests for configuring the RocksDB State Backend + * Tests for configuring the RocksDB State Backend. */ @SuppressWarnings("serial") public class RocksDBStateBackendConfigTest { @@ -102,7 +103,6 @@ public void testSetDbPath() throws Exception { new KeyGroupRange(0, 0), env.getTaskKvStateRegistry()); - File instanceBasePath = keyedBackend.getInstanceBasePath(); assertThat(instanceBasePath.getAbsolutePath(), anyOf(startsWith(testDir1.getAbsolutePath()), startsWith(testDir2.getAbsolutePath()))); @@ -158,7 +158,6 @@ public void testUseTempDirectories() throws Exception { new KeyGroupRange(0, 0), env.getTaskKvStateRegistry()); - File instanceBasePath = keyedBackend.getInstanceBasePath(); assertThat(instanceBasePath.getAbsolutePath(), anyOf(startsWith(dir1.getAbsolutePath()), startsWith(dir2.getAbsolutePath()))); } diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactoryTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactoryTest.java index 9eb662aebaa288..5a937c44f24af0 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactoryTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendFactoryTest.java @@ -22,6 +22,9 @@ import static org.junit.Assert.assertEquals; +/** + * Tests for the RocksDBStateBackendFactory. + */ public class RocksDBStateBackendFactoryTest { @Test diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java index 8d0db6984ff6e5..8b44a475de1832 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackendTest.java @@ -18,8 +18,6 @@ package org.apache.flink.contrib.streaming.state; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.state.ValueState; @@ -42,6 +40,9 @@ import org.apache.flink.runtime.state.VoidNamespaceSerializer; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.util.BlockerCheckpointStreamFactory; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.IOFileFilter; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -372,7 +373,6 @@ public void testSharedIncrementalStateDeRegistration() throws Exception { ValueState state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId); - Queue previousStateHandles = new LinkedList<>(); SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry()); for (int checkpointId = 0; checkpointId < 3; ++checkpointId) { diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDbMultiClassLoaderTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDbMultiClassLoaderTest.java index c53fa3ea535a92..4ec65324fcd1b4 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDbMultiClassLoaderTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDbMultiClassLoaderTest.java @@ -27,7 +27,7 @@ import java.net.URL; import java.net.URLClassLoader; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNotEquals; /** * This test validates that the RocksDB JNI library loading works properly @@ -60,7 +60,7 @@ public void testTwoSeparateClassLoaders() throws Exception { final String tempDir = tmp.newFolder().getAbsolutePath(); - final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); + final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class); meth1.setAccessible(true); meth2.setAccessible(true); diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java index 71475830d9baf9..3231e9659eeb63 100644 --- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java +++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java @@ -26,7 +26,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import org.rocksdb.CompactionStyle; import org.rocksdb.NativeLibraryLoader; import org.rocksdb.Options; @@ -47,7 +46,7 @@ public class RocksDBPerformanceTest extends TestLogger { @Rule - public final TemporaryFolder TMP = new TemporaryFolder(); + public final TemporaryFolder tmp = new TemporaryFolder(); @Rule public final RetryRule retry = new RetryRule(); @@ -55,7 +54,7 @@ public class RocksDBPerformanceTest extends TestLogger { @Test(timeout = 2000) @RetryOnFailure(times = 3) public void testRocksDbMergePerformance() throws Exception { - final File rocksDir = TMP.newFolder(); + final File rocksDir = tmp.newFolder(); // ensure the RocksDB library is loaded to a distinct location each retry NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath()); @@ -83,8 +82,8 @@ public void testRocksDbMergePerformance() throws Exception { .setSync(false) .setDisableWAL(true); - final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) - { + final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) { + // ----- insert ----- log.info("begin insert"); @@ -133,7 +132,7 @@ public void testRocksDbMergePerformance() throws Exception { @Test(timeout = 2000) @RetryOnFailure(times = 3) public void testRocksDbRangeGetPerformance() throws Exception { - final File rocksDir = TMP.newFolder(); + final File rocksDir = tmp.newFolder(); // ensure the RocksDB library is loaded to a distinct location each retry NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath()); @@ -161,8 +160,8 @@ public void testRocksDbRangeGetPerformance() throws Exception { .setSync(false) .setDisableWAL(true); - final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) - { + final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) { + final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4); final Unsafe unsafe = MemoryUtils.UNSAFE; @@ -205,7 +204,6 @@ public void testRocksDbRangeGetPerformance() throws Exception { } } - private static boolean samePrefix(byte[] prefix, byte[] key) { for (int i = 0; i < prefix.length; i++) { if (prefix[i] != key [i]) { From 9abcc6750f8f8158a48ec705fdc16473b6175007 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 23 May 2017 22:41:09 +0200 Subject: [PATCH 3/5] [FLINK-6695] Activate strict checkstyle for flink-storm --- flink-contrib/flink-storm/pom.xml | 39 ++++++++++++ .../apache/flink/storm/api/FlinkClient.java | 60 ++++++++++--------- .../flink/storm/api/FlinkLocalCluster.java | 28 ++++----- .../storm/api/FlinkOutputFieldsDeclarer.java | 17 +++--- .../flink/storm/api/FlinkSubmitter.java | 27 +++++---- .../apache/flink/storm/api/FlinkTopology.java | 37 ++++++------ .../storm/api/StormFlinkStreamMerger.java | 5 +- .../storm/api/TwoFlinkStreamsMerger.java | 5 +- .../apache/flink/storm/util/FiniteSpout.java | 2 +- .../storm/util/NullTerminatingSpout.java | 11 ++-- .../flink/storm/util/SplitStreamMapper.java | 3 +- .../flink/storm/util/SplitStreamType.java | 1 + .../util/SpoutOutputCollectorObserver.java | 9 +-- .../apache/flink/storm/util/StormConfig.java | 7 ++- .../flink/storm/util/StormStreamSelector.java | 3 +- .../wrappers/AbstractStormCollector.java | 7 ++- .../flink/storm/wrappers/BoltCollector.java | 11 ++-- .../flink/storm/wrappers/BoltWrapper.java | 44 +++++++------- .../storm/wrappers/FlinkTopologyContext.java | 2 +- .../wrappers/MergedInputsBoltWrapper.java | 12 ++-- .../flink/storm/wrappers/SpoutCollector.java | 9 ++- .../flink/storm/wrappers/SpoutWrapper.java | 36 +++++------ .../flink/storm/wrappers/StormTuple.java | 2 +- .../storm/wrappers/WrapperSetupHelper.java | 22 +++---- .../api/FlinkOutputFieldsDeclarerTest.java | 9 ++- .../flink/storm/api/FlinkTopologyTest.java | 12 ++-- .../org/apache/flink/storm/api/TestBolt.java | 4 ++ .../org/apache/flink/storm/api/TestSpout.java | 4 ++ .../apache/flink/storm/util/AbstractTest.java | 3 + .../flink/storm/util/FiniteTestSpout.java | 3 + .../storm/util/NullTerminatingSpoutTest.java | 17 +++--- .../SpoutOutputCollectorObserverTest.java | 13 ++-- .../storm/util/StormStreamSelectorTest.java | 4 ++ .../flink/storm/util/TestDummyBolt.java | 16 +++-- .../flink/storm/util/TestDummySpout.java | 8 ++- .../org/apache/flink/storm/util/TestSink.java | 12 ++-- .../storm/wrappers/BoltCollectorTest.java | 7 ++- .../flink/storm/wrappers/BoltWrapperTest.java | 21 ++++--- .../wrappers/FlinkTopologyContextTest.java | 6 +- .../SetupOutputFieldsDeclarerTest.java | 6 +- .../storm/wrappers/SpoutCollectorTest.java | 7 ++- .../storm/wrappers/SpoutWrapperTest.java | 12 ++-- .../flink/storm/wrappers/StormTupleTest.java | 19 +++--- .../wrappers/WrapperSetupHelperTest.java | 4 ++ .../WrapperSetupInLocalClusterTest.java | 36 ++++++----- .../src/test/resources/log4j-test.properties | 2 +- 46 files changed, 369 insertions(+), 255 deletions(-) diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml index a10ff68bc3d849..1baf26b342ff93 100644 --- a/flink-contrib/flink-storm/pom.xml +++ b/flink-contrib/flink-storm/pom.xml @@ -181,5 +181,44 @@ under the License. + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + /tools/maven/strict-checkstyle.xml + /tools/maven/suppressions.xml + true + true + true + + + + + test-compile + + check + + + + + + diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java index 626335d948e2f9..88a38e209947b5 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkClient.java @@ -18,21 +18,6 @@ package org.apache.flink.storm.api; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.pattern.Patterns; -import akka.util.Timeout; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution; -import org.apache.flink.runtime.jobmaster.JobMaster; -import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; -import org.apache.storm.Config; -import org.apache.storm.generated.AlreadyAliveException; -import org.apache.storm.generated.InvalidTopologyException; -import org.apache.storm.generated.KillOptions; -import org.apache.storm.generated.NotAliveException; -import org.apache.storm.utils.NimbusClient; -import org.apache.storm.utils.Utils; -import com.esotericsoftware.kryo.Serializer; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.client.program.ClusterClient; @@ -46,17 +31,29 @@ import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.client.JobStatusMessage; +import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.AddressResolution; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobmaster.JobMaster; import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.messages.JobManagerMessages.RunningJobsStatus; +import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; import org.apache.flink.storm.util.StormConfig; import org.apache.flink.streaming.api.graph.StreamGraph; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.pattern.Patterns; +import akka.util.Timeout; +import com.esotericsoftware.kryo.Serializer; +import org.apache.storm.Config; +import org.apache.storm.generated.AlreadyAliveException; +import org.apache.storm.generated.InvalidTopologyException; +import org.apache.storm.generated.KillOptions; +import org.apache.storm.generated.NotAliveException; +import org.apache.storm.utils.NimbusClient; +import org.apache.storm.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Some; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.FiniteDuration; import java.io.File; import java.io.IOException; @@ -67,6 +64,11 @@ import java.util.Map; import java.util.Map.Entry; +import scala.Some; +import scala.concurrent.Await; +import scala.concurrent.Future; +import scala.concurrent.duration.FiniteDuration; + /** * {@link FlinkClient} mimics a Storm {@link NimbusClient} and {@link Nimbus}{@code .Client} at once, to interact with * Flink's JobManager instead of Storm's Nimbus. @@ -76,13 +78,13 @@ public class FlinkClient { /** The log used by this client. */ private static final Logger LOG = LoggerFactory.getLogger(FlinkClient.class); - /** The client's configuration */ - private final Map conf; - /** The jobmanager's host name */ + /** The client's configuration. */ + private final Map conf; + /** The jobmanager's host name. */ private final String jobManagerHost; - /** The jobmanager's rpc port */ + /** The jobmanager's rpc port. */ private final int jobManagerPort; - /** The user specified timeout in milliseconds */ + /** The user specified timeout in milliseconds. */ private final String timeout; // The following methods are derived from "backtype.storm.utils.NimbusClient" @@ -145,8 +147,8 @@ public static FlinkClient getConfiguredClient(final Map conf) { /** * Return a reference to itself. - *

    - * {@link FlinkClient} mimics both, {@link NimbusClient} and {@link Nimbus}{@code .Client}, at once. + * + *

    {@link FlinkClient} mimics both, {@link NimbusClient} and {@link Nimbus}{@code .Client}, at once. * * @return A reference to itself. */ @@ -188,7 +190,7 @@ public void submitTopologyWithOpts(final String name, final String uploadedJarLo try { FlinkClient.addStormConfigToTopology(topology, conf); - } catch(ClassNotFoundException e) { + } catch (ClassNotFoundException e) { LOG.error("Could not register class for Kryo serialization.", e); throw new InvalidTopologyException("Could not register class for Kryo serialization."); } @@ -352,9 +354,9 @@ static void addStormConfigToTopology(FlinkTopology topology, Map conf) throws Cl if (klass instanceof String) { flinkConfig.registerKryoType(Class.forName((String) klass)); } else { - for (Entry register : ((Map)klass).entrySet()) { + for (Entry register : ((Map) klass).entrySet()) { flinkConfig.registerTypeWithKryoSerializer(Class.forName(register.getKey()), - (Class>)Class.forName(register.getValue())); + (Class>) Class.forName(register.getValue())); } } } diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java index d69d3459692049..364c4d54c0c363 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkLocalCluster.java @@ -18,7 +18,14 @@ package org.apache.flink.storm.api; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.minicluster.FlinkMiniCluster; +import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; +import org.apache.flink.streaming.api.graph.StreamGraph; + import org.apache.storm.LocalCluster; import org.apache.storm.generated.ClusterSummary; import org.apache.storm.generated.KillOptions; @@ -26,13 +33,6 @@ import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.SubmitOptions; import org.apache.storm.generated.TopologyInfo; - -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.minicluster.FlinkMiniCluster; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; -import org.apache.flink.streaming.api.graph.StreamGraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,17 +44,15 @@ */ public class FlinkLocalCluster { - /** The log used by this mini cluster */ + /** The log used by this mini cluster. */ private static final Logger LOG = LoggerFactory.getLogger(FlinkLocalCluster.class); - /** The Flink mini cluster on which to execute the programs */ + /** The Flink mini cluster on which to execute the programs. */ private FlinkMiniCluster flink; /** Configuration key to submit topology in blocking mode if flag is set to {@code true}. */ public static final String SUBMIT_BLOCKING = "SUBMIT_STORM_TOPOLOGY_BLOCKING"; - - public FlinkLocalCluster() { } @@ -62,8 +60,6 @@ public FlinkLocalCluster(FlinkMiniCluster flink) { this.flink = Objects.requireNonNull(flink); } - - @SuppressWarnings("rawtypes") public void submitTopology(final String topologyName, final Map conf, final FlinkTopology topology) throws Exception { @@ -77,8 +73,8 @@ public void submitTopologyWithOpts(final String topologyName, final Map conf, fi boolean submitBlocking = false; if (conf != null) { Object blockingFlag = conf.get(SUBMIT_BLOCKING); - if(blockingFlag != null && blockingFlag instanceof Boolean) { - submitBlocking = ((Boolean)blockingFlag).booleanValue(); + if (blockingFlag != null && blockingFlag instanceof Boolean) { + submitBlocking = ((Boolean) blockingFlag).booleanValue(); } } @@ -184,7 +180,7 @@ public static void initialize(LocalClusterFactory clusterFactory) { /** * A factory that creates local clusters. */ - public static interface LocalClusterFactory { + public interface LocalClusterFactory { /** * Creates a local Flink cluster. diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java index b0bebefce8202f..b1e8a47d8fd7a8 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarer.java @@ -17,13 +17,14 @@ package org.apache.flink.storm.api; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.typeutils.TypeExtractor; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.utils.Utils; + import java.util.HashMap; import java.util.List; @@ -80,13 +81,13 @@ public void declareStream(final String streamId, final boolean direct, final Fie /** * Returns {@link TypeInformation} for the declared output schema for a specific stream. - * + * * @param streamId * A stream ID. - * + * * @return output type information for the declared output schema of the specified stream; or {@code null} if * {@code streamId == null} - * + * * @throws IllegalArgumentException * If no output schema was declared for the specified stream or if more then 25 attributes got declared. */ @@ -143,12 +144,12 @@ public int compareTo(final DefaultComparable o) { /** * Computes the indexes within the declared output schema of the specified stream, for a list of given * field-grouping attributes. - * + * * @param streamId * A stream ID. * @param groupingFields * The names of the key fields. - * + * * @return array of {@code int}s that contains the index within the output schema for each attribute in the given * list */ diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java index 3b191b0d09ff2a..6135d4db962877 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkSubmitter.java @@ -14,34 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.api; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.client.program.ContextEnvironment; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.GlobalConfiguration; + import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.generated.AlreadyAliveException; import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.generated.SubmitOptions; import org.apache.storm.utils.Utils; - -import java.net.URISyntaxException; -import java.net.URL; -import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.client.program.ContextEnvironment; -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.GlobalConfiguration; import org.json.simple.JSONValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; import java.util.Map; /** * {@link FlinkSubmitter} mimics a {@link StormSubmitter} to submit Storm topologies to a Flink cluster. */ public class FlinkSubmitter { - public final static Logger logger = LoggerFactory.getLogger(FlinkSubmitter.class); + private static final Logger LOG = LoggerFactory.getLogger(FlinkSubmitter.class); /** * Submits a topology to run on the cluster. A topology runs forever or until explicitly killed. @@ -121,17 +122,17 @@ public static void submitTopology(final String name, final Map stormConf, final } } - logger.info("Submitting topology " + name + " in distributed mode with conf " + serConf); + LOG.info("Submitting topology " + name + " in distributed mode with conf " + serConf); client.submitTopologyWithOpts(name, localJar, topology); } catch (final InvalidTopologyException e) { - logger.warn("Topology submission exception: " + e.get_msg()); + LOG.warn("Topology submission exception: " + e.get_msg()); throw e; } catch (final AlreadyAliveException e) { - logger.warn("Topology already alive exception", e); + LOG.warn("Topology already alive exception", e); throw e; } - logger.info("Finished submitting topology: " + name); + LOG.info("Finished submitting topology: " + name); } /** diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java index 2b36feb63cd940..3b78a9089aa508 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/FlinkTopology.java @@ -16,17 +16,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.api; -import org.apache.storm.generated.ComponentCommon; -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.generated.Grouping; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.IRichStateSpout; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; +package org.apache.flink.storm.api; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -46,6 +37,16 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.InstantiationUtil; +import org.apache.storm.generated.ComponentCommon; +import org.apache.storm.generated.GlobalStreamId; +import org.apache.storm.generated.Grouping; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.topology.IRichBolt; +import org.apache.storm.topology.IRichSpout; +import org.apache.storm.topology.IRichStateSpout; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.tuple.Fields; + import java.io.IOException; import java.lang.reflect.Field; import java.util.HashMap; @@ -62,9 +63,9 @@ */ public class FlinkTopology { - /** All declared streams and output schemas by operator ID */ + /** All declared streams and output schemas by operator ID. */ private final HashMap> outputStreams = new HashMap>(); - /** All spouts&bolts declarers by their ID */ + /** All spouts&bolts declarers by their ID. */ private final HashMap declarers = new HashMap(); private final HashMap>> unprocessdInputsPerBolt = @@ -96,7 +97,6 @@ private FlinkTopology(TopologyBuilder builder) { } /** - * * Creates a Flink program that uses the specified spouts and bolts. * @param stormBuilder The Storm topology builder to use for creating the Flink topology. * @return A {@link FlinkTopology} which contains the translated Storm topology and may be executed. @@ -123,7 +123,6 @@ public JobExecutionResult execute() throws Exception { return env.execute(); } - @SuppressWarnings("unchecked") private Map getPrivateField(String field) { try { @@ -161,18 +160,16 @@ private void translateTopology() { /* Translation of topology */ - for (final Entry spout : spouts.entrySet()) { final String spoutId = spout.getKey(); final IRichSpout userSpout = spout.getValue(); final FlinkOutputFieldsDeclarer declarer = new FlinkOutputFieldsDeclarer(); userSpout.declareOutputFields(declarer); - final HashMap sourceStreams = declarer.outputStreams; + final HashMap sourceStreams = declarer.outputStreams; this.outputStreams.put(spoutId, sourceStreams); declarers.put(spoutId, declarer); - final HashMap> outputStreams = new HashMap>(); final DataStreamSource source; @@ -222,7 +219,7 @@ private void translateTopology() { * 1. Connect all spout streams with bolts streams * 2. Then proceed with the bolts stream already connected * - * Because we do not know the order in which an iterator steps over a set, we might process a consumer before + *

    Because we do not know the order in which an iterator steps over a set, we might process a consumer before * its producer * ->thus, we might need to repeat multiple times */ @@ -418,7 +415,7 @@ private SingleOutputStreamOperator createOutput(String boltId, IRichBolt bolt final SingleOutputStreamOperator outStream; // only one input - if(inputStreams.entrySet().size() == 1) { + if (inputStreams.entrySet().size() == 1) { BoltWrapper boltWrapper = new BoltWrapper<>(bolt, boltId, inputStreamId1, inputComponentId1, inputSchema1, null); boltWrapper.setStormTopology(stormTopology); @@ -444,7 +441,7 @@ private SingleOutputStreamOperator createOutput(String boltId, IRichBolt bolt final SingleOutputStreamOperator> multiStream; // only one input - if(inputStreams.entrySet().size() == 1) { + if (inputStreams.entrySet().size() == 1) { final BoltWrapper> boltWrapperMultipleOutputs = new BoltWrapper<>( bolt, boltId, inputStreamId1, inputComponentId1, inputSchema1, null); boltWrapperMultipleOutputs.setStormTopology(stormTopology); diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java index 160a7d9ea703b6..00c467e3da7a2d 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/StormFlinkStreamMerger.java @@ -3,13 +3,14 @@ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ + package org.apache.flink.storm.api; import org.apache.flink.storm.wrappers.StormTuple; diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java index 1e6e2ed3b5aa7e..f7bcb12c8110ba 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/api/TwoFlinkStreamsMerger.java @@ -3,13 +3,14 @@ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ + package org.apache.flink.storm.api; import org.apache.flink.storm.wrappers.StormTuple; diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java index 10f9797baa427d..7615b2e606367f 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/FiniteSpout.java @@ -31,6 +31,6 @@ public interface FiniteSpout extends IRichSpout { * * @return true, if the spout's stream reached its end, false otherwise */ - public boolean reachedEnd(); + boolean reachedEnd(); } diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java index 20e3309590cfd6..a830b105403594 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/NullTerminatingSpout.java @@ -15,15 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.util; -import java.util.Map; +package org.apache.flink.storm.util; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IRichSpout; import org.apache.storm.topology.OutputFieldsDeclarer; +import java.util.Map; + /** * {@link NullTerminatingSpout} in a finite spout (ie, implements {@link FiniteSpout} interface) that wraps an * infinite spout, and returns {@code true} in {@link #reachedEnd()} when the wrapped spout does not emit a tuple @@ -37,14 +38,10 @@ public class NullTerminatingSpout implements FiniteSpout { /** The observer that checks if the given spouts emit a tuple or not on nextTuple(). */ private SpoutOutputCollectorObserver observer; - - public NullTerminatingSpout(IRichSpout spout) { this.spout = spout; } - - @Override public void open(@SuppressWarnings("rawtypes") Map conf, TopologyContext context, SpoutOutputCollector collector) { this.observer = new SpoutOutputCollectorObserver(collector); @@ -95,7 +92,7 @@ public Map getComponentConfiguration() { @Override public boolean reachedEnd() { - return this.observer.emitted == false; + return !this.observer.emitted; } } diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java index 1fb5e024e175cc..d2e84c5c2a5bd0 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamMapper.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.flink.api.common.functions.MapFunction; @@ -25,7 +26,7 @@ * a "clean" stream from a Spout/Bolt that declared multiple output streams (after the streams got separated using * {@link DataStream#split(org.apache.flink.streaming.api.collector.selector.OutputSelector) .split(...)} and * {@link SplitStream#select(String...) .select(...)}). - * + * * @param */ public class SplitStreamMapper implements MapFunction, T> { diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java index 5056795200d903..36894c793490c7 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SplitStreamType.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.flink.streaming.api.datastream.DataStream; diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java index 9e222ecee68f74..8be466eb98b614 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/SpoutOutputCollectorObserver.java @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.util; -import java.util.List; +package org.apache.flink.storm.util; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.utils.Utils; +import java.util.List; + /** * Observes if a call to any {@code emit(...)} or {@code emitDirect(...)} method is made. * The internal flag {@link #emitted} must be reset by the user manually. @@ -33,15 +34,11 @@ class SpoutOutputCollectorObserver extends SpoutOutputCollector { /** The internal flag that it set to {@code true} if a tuple gets emitted. */ boolean emitted; - - public SpoutOutputCollectorObserver(SpoutOutputCollector delegate) { super(null); this.delegate = delegate; } - - @Override public List emit(String streamId, List tuple, Object messageId) { emitted = true; diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java index 040c395936dd5a..a9d7bfd7abb7f2 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormConfig.java @@ -15,11 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; -import org.apache.storm.Config; import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; +import org.apache.storm.Config; + import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -44,7 +46,7 @@ public StormConfig() { /** * Creates an configuration with initial values provided by the given {@code Map}. - * + * * @param config * Initial values for this configuration. */ @@ -53,7 +55,6 @@ public StormConfig(Map config) { this.config.putAll(config); } - @Override public int size() { return this.config.size(); diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java index 6072e0f558db76..33ba3748a7aeaf 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/util/StormStreamSelector.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.flink.storm.api.FlinkTopology; @@ -45,4 +46,4 @@ public Iterable select(SplitStreamType value) { return streamId; } -} \ No newline at end of file +} diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java index 7f4d7d1853cdcb..21ce1152387b2b 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/AbstractStormCollector.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.wrappers; import org.apache.flink.api.java.tuple.Tuple; @@ -50,7 +51,7 @@ abstract class AbstractStormCollector { * Instantiates a new {@link AbstractStormCollector} that emits Flink tuples via {@link #doEmit(Object)}. If the * number of attributes is negative, any output type is supported (ie, raw type). If the number of attributes is * between 0 and 25, the output type is {@link Tuple0} to {@link Tuple25}, respectively. - * + * * @param numberOfAttributes * The number of attributes of the emitted tuples per output stream. * @param taskId @@ -107,7 +108,7 @@ abstract class AbstractStormCollector { /** * Transforms a Storm tuple into a Flink tuple of type {@code OUT} and emits this tuple via {@link #doEmit(Object)} * to the specified output stream. - * + * * @param The * The output stream id. * @param tuple @@ -160,7 +161,7 @@ protected final List tansformAndEmit(final String streamId, final List< /** * Emits a Flink tuple. - * + * * @param flinkTuple * The tuple to be emitted. * @return the IDs of the tasks this tuple was sent to diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java index 7b947079bb6e60..82c7be31adf2c1 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltCollector.java @@ -17,14 +17,14 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.task.IOutputCollector; -import org.apache.storm.tuple.Tuple; - import org.apache.flink.api.java.tuple.Tuple0; import org.apache.flink.api.java.tuple.Tuple25; import org.apache.flink.streaming.api.operators.Output; import org.apache.flink.util.Collector; +import org.apache.storm.task.IOutputCollector; +import org.apache.storm.tuple.Tuple; + import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -36,14 +36,14 @@ */ class BoltCollector extends AbstractStormCollector implements IOutputCollector { - /** The Flink output Collector */ + /** The Flink output Collector. */ private final Collector flinkOutput; /** * Instantiates a new {@link BoltCollector} that emits Flink tuples to the given Flink output object. If the * number of attributes is negative, any output type is supported (ie, raw type). If the number of attributes is * between 0 and 25, the output type is {@link Tuple0} to {@link Tuple25}, respectively. - * + * * @param numberOfAttributes * The number of attributes of the emitted tuples per output stream. * @param taskId @@ -91,5 +91,4 @@ public void fail(final Tuple input) {} @Override public void resetTimeout(Tuple var1) {} - } diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java index 731f28feb5e4e4..590faf382dbe34 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/BoltWrapper.java @@ -17,16 +17,6 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.generated.Grouping; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.utils.Utils; - import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple0; @@ -38,6 +28,16 @@ import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.storm.generated.GlobalStreamId; +import org.apache.storm.generated.Grouping; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.IRichBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.MessageId; +import org.apache.storm.utils.Utils; + import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -57,9 +57,9 @@ public class BoltWrapper extends AbstractStreamOperator implements private static final long serialVersionUID = -4788589118464155835L; /** The default input component ID. */ - public final static String DEFAULT_ID = "default ID"; + public static final String DEFAULT_ID = "default ID"; /** The default bolt ID. */ - public final static String DEFUALT_BOLT_NAME = "Unnamed Bolt"; + public static final String DEFUALT_BOLT_NAME = "Unnamed Bolt"; /** The wrapped Storm {@link IRichBolt bolt}. */ protected final IRichBolt bolt; @@ -90,7 +90,7 @@ public class BoltWrapper extends AbstractStreamOperator implements * within a Flink streaming program. As no input schema is defined, attribute-by-name access in only possible for * POJO input types. The output type will be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's * declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @throws IllegalArgumentException @@ -105,7 +105,7 @@ public BoltWrapper(final IRichBolt bolt) throws IllegalArgumentException { * within a Flink streaming program. The given input schema enable attribute-by-name access for input types * {@link Tuple0} to {@link Tuple25}. The output type will be one of {@link Tuple0} to {@link Tuple25} depending on * the bolt's declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param inputSchema @@ -124,7 +124,7 @@ public BoltWrapper(final IRichBolt bolt, final Fields inputSchema) * POJO input types. The output type can be any type if parameter {@code rawOutput} is {@code true} and the bolt's * number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will be one of * {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param rawOutputs @@ -135,7 +135,7 @@ public BoltWrapper(final IRichBolt bolt, final Fields inputSchema) * {@code rawOuput} is {@code false} and the number of declared output attributes is not within range * [1;25]. */ - public BoltWrapper(final IRichBolt bolt, final String[] rawOutputs) + public BoltWrapper(final IRichBolt bolt, final String[] rawOutputs) throws IllegalArgumentException { this(bolt, null, asList(rawOutputs)); } @@ -146,7 +146,7 @@ public BoltWrapper(final IRichBolt bolt, final String[] rawOutputs) * POJO input types. The output type can be any type if parameter {@code rawOutput} is {@code true} and the bolt's * number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will be one of * {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param rawOutputs @@ -167,7 +167,7 @@ public BoltWrapper(final IRichBolt bolt, final Collection rawOutputs) th * {@link Tuple0} to {@link Tuple25}. The output type can be any type if parameter {@code rawOutput} is {@code true} * and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will * be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param inputSchema @@ -183,7 +183,7 @@ public BoltWrapper(final IRichBolt bolt, final Collection rawOutputs) th public BoltWrapper( final IRichBolt bolt, final Fields inputSchema, - final String[] rawOutputs) + final String[] rawOutputs) throws IllegalArgumentException { this(bolt, inputSchema, asList(rawOutputs)); } @@ -194,7 +194,7 @@ public BoltWrapper( * {@link Tuple0} to {@link Tuple25}. The output type can be any type if parameter {@code rawOutput} is {@code true} * and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will * be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param inputSchema @@ -220,7 +220,7 @@ public BoltWrapper(final IRichBolt bolt, final Fields inputSchema, * {@link Tuple0} to {@link Tuple25}. The output type can be any type if parameter {@code rawOutput} is {@code true} * and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the output type will * be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param name @@ -244,7 +244,7 @@ public BoltWrapper(final IRichBolt bolt, final String name, final String inputSt /** * Sets the original Storm topology. - * + * * @param stormTopology * The original Storm topology. */ diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java index f55f0e33a9f952..e84abccefbab7f 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/FlinkTopologyContext.java @@ -17,6 +17,7 @@ package org.apache.flink.storm.wrappers; +import clojure.lang.Atom; import org.apache.storm.generated.StormTopology; import org.apache.storm.hooks.ITaskHook; import org.apache.storm.metric.api.CombinedMetric; @@ -27,7 +28,6 @@ import org.apache.storm.state.ISubscribedState; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Fields; -import clojure.lang.Atom; import java.util.Collection; import java.util.List; diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java index 6dd6973691a1b2..07abffc80937c1 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/MergedInputsBoltWrapper.java @@ -17,13 +17,13 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.topology.IRichBolt; - import org.apache.flink.api.java.tuple.Tuple0; import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.api.java.tuple.Tuple25; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.storm.topology.IRichBolt; + import java.util.Collection; import static java.util.Arrays.asList; @@ -40,7 +40,7 @@ public final class MergedInputsBoltWrapper extends BoltWrapper ra * {@code true} and the bolt's number of declared output tuples is 1. If {@code rawOutput} is {@code false} the * output type will be one of {@link Tuple0} to {@link Tuple25} depending on the bolt's declared number of * attributes. - * + * * @param bolt * The Storm {@link IRichBolt bolt} to be used. * @param name diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java index 5404027d99221f..6e3a39af6abceb 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutCollector.java @@ -17,11 +17,12 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.spout.ISpoutOutputCollector; import org.apache.flink.api.java.tuple.Tuple0; import org.apache.flink.api.java.tuple.Tuple25; import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; +import org.apache.storm.spout.ISpoutOutputCollector; + import java.util.HashMap; import java.util.List; @@ -32,14 +33,14 @@ */ class SpoutCollector extends AbstractStormCollector implements ISpoutOutputCollector { - /** The Flink source context object */ + /** The Flink source context object. */ private final SourceContext flinkContext; /** * Instantiates a new {@link SpoutCollector} that emits Flink tuples to the given Flink source context. If the * number of attributes is specified as zero, any output type is supported. If the number of attributes is between 0 * to 25, the output type is {@link Tuple0} to {@link Tuple25}, respectively. - * + * * @param numberOfAttributes * The number of attributes of the emitted tuples. * @param taskId @@ -73,7 +74,6 @@ public List emit(final String streamId, final List tuple, final return this.tansformAndEmit(streamId, tuple); } - @Override public void emitDirect(final int taskId, final String streamId, final List tuple, final Object messageId) { throw new UnsupportedOperationException("Direct emit is not supported by Flink"); @@ -83,5 +83,4 @@ public long getPendingCount() { return 0; } - } diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java index 3dd1e1008ef2a3..458fffbfc37ea8 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/SpoutWrapper.java @@ -17,11 +17,6 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; - import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters; import org.apache.flink.api.common.functions.StoppableFunction; import org.apache.flink.api.java.tuple.Tuple0; @@ -32,6 +27,11 @@ import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.IRichSpout; + import java.util.Collection; import java.util.HashMap; @@ -73,7 +73,7 @@ public final class SpoutWrapper extends RichParallelSourceFunction imp * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of * the given {@link IRichSpout spout} in an infinite loop. The output type will be one of {@link Tuple0} to * {@link Tuple25} depending on the spout's declared number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @throws IllegalArgumentException @@ -87,7 +87,7 @@ public SpoutWrapper(final IRichSpout spout) throws IllegalArgumentException { * Instantiates a new {@link SpoutWrapper} that calls the {@link IRichSpout#nextTuple() nextTuple()} method of * the given {@link IRichSpout spout} a finite number of times. The output type will be one of {@link Tuple0} to * {@link Tuple25} depending on the spout's declared number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @param numberOfInvocations @@ -108,7 +108,7 @@ public SpoutWrapper(final IRichSpout spout, final Integer numberOfInvocations) * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared * number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @param rawOutputs @@ -130,7 +130,7 @@ public SpoutWrapper(final IRichSpout spout, final String[] rawOutputs) * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared * number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @param rawOutputs @@ -156,7 +156,7 @@ public SpoutWrapper(final IRichSpout spout, final String[] rawOutputs, * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared * number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @param rawOutputs @@ -178,7 +178,7 @@ public SpoutWrapper(final IRichSpout spout, final Collection rawOutputs) * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared * number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @param rawOutputs @@ -204,7 +204,7 @@ public SpoutWrapper(final IRichSpout spout, final Collection rawOutputs, * {@code rawOutput} is {@code true} and the spout's number of declared output tuples is 1. If {@code rawOutput} is * {@code false} the output type will be one of {@link Tuple0} to {@link Tuple25} depending on the spout's declared * number of attributes. - * + * * @param spout * The {@link IRichSpout spout} to be used. * @param name @@ -231,7 +231,7 @@ public SpoutWrapper(final IRichSpout spout, final String name, final Collection< /** * Sets the original Storm topology. - * + * * @param stormTopology * The original Storm topology. */ @@ -240,7 +240,7 @@ public void setStormTopology(StormTopology stormTopology) { } @Override - public final void run(final SourceContext ctx) throws Exception { + public void run(final SourceContext ctx) throws Exception { final GlobalJobParameters config = super.getRuntimeContext().getExecutionConfig() .getGlobalJobParameters(); StormConfig stormConfig = new StormConfig(); @@ -292,8 +292,8 @@ public final void run(final SourceContext ctx) throws Exception { /** * {@inheritDoc} - *

    - * Sets the {@link #isRunning} flag to {@code false}. + * + *

    Sets the {@link #isRunning} flag to {@code false}. */ @Override public void cancel() { @@ -302,8 +302,8 @@ public void cancel() { /** * {@inheritDoc} - *

    - * Sets the {@link #isRunning} flag to {@code false}. + * + *

    Sets the {@link #isRunning} flag to {@code false}. */ @Override public void stop() { diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java index 30085fcd176cb7..a1d33e80baa789 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/StormTuple.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.wrappers; /* @@ -52,7 +53,6 @@ public class StormTuple implements org.apache.storm.tuple.Tuple { /** The message that is associated with this tuple. */ private final MessageId messageId; - /** * Create a new Storm tuple from the given Flink tuple. * diff --git a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java index 3a9b650d792a3f..16112114c18562 100644 --- a/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java +++ b/flink-contrib/flink-storm/src/main/java/org/apache/flink/storm/wrappers/WrapperSetupHelper.java @@ -14,8 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.wrappers; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; + +import clojure.lang.Atom; import org.apache.storm.Config; import org.apache.storm.generated.Bolt; import org.apache.storm.generated.ComponentCommon; @@ -28,8 +32,6 @@ import org.apache.storm.topology.IRichBolt; import org.apache.storm.topology.IRichSpout; import org.apache.storm.tuple.Fields; -import clojure.lang.Atom; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import java.util.ArrayList; import java.util.Collection; @@ -45,14 +47,14 @@ class WrapperSetupHelper { /** The configuration key for the topology name. */ - final static String TOPOLOGY_NAME = "storm.topology.name"; + static final String TOPOLOGY_NAME = "storm.topology.name"; /** * Computes the number of output attributes used by a {@link SpoutWrapper} or {@link BoltWrapper} * per declared output stream. The number is {@code -1} for raw output type or a value within range [0;25] for * output type {@link org.apache.flink.api.java.tuple.Tuple0 Tuple0} to * {@link org.apache.flink.api.java.tuple.Tuple25 Tuple25}. - * + * * @param spoutOrBolt * The Storm {@link IRichSpout spout} or {@link IRichBolt bolt} to be used. * @param rawOutputs @@ -96,7 +98,7 @@ static HashMap getNumberOfAttributes(final IComponent spoutOrBo /** * Creates a {@link TopologyContext} for a Spout or Bolt instance (ie, Flink task / Storm executor). - * + * * @param context * The Flink runtime context. * @param spoutOrBolt @@ -203,7 +205,7 @@ static synchronized TopologyContext createTopologyContext( /** * Sets up {@code taskToComponents}, {@code componentToSortedTasks}, and {@code componentToStreamToFields} for a * single instance of a Spout or Bolt (ie, task or executor). Furthermore, is computes the unique task-id. - * + * * @param componentId * The ID of the Spout/Bolt in the topology. * @param common @@ -220,7 +222,7 @@ static synchronized TopologyContext createTopologyContext( * OUTPUT: A map from all component IDs to their sorted list of corresponding task IDs. * @param componentToStreamToFields * OUTPUT: A map from all component IDs to there output streams and output fields. - * + * * @return A unique task ID if the currently processed Spout or Bolt ({@code componentId}) is equal to the current * Flink operator {@code operatorName} -- {@code null} otherwise. */ @@ -229,7 +231,7 @@ private static Integer processSingleOperator(final String componentId, final int dop, final Map taskToComponents, final Map> componentToSortedTasks, final Map> componentToStreamToFields) { - final int parallelism_hint = common.get_parallelism_hint(); + final int parallelismHint = common.get_parallelism_hint(); Integer taskId = null; if (componentId.equals(operatorName)) { @@ -237,7 +239,7 @@ private static Integer processSingleOperator(final String componentId, } List sortedTasks = new ArrayList(dop); - for (int i = 0; i < parallelism_hint; ++i) { + for (int i = 0; i < parallelismHint; ++i) { taskToComponents.put(tid, componentId); sortedTasks.add(tid); ++tid; @@ -245,7 +247,7 @@ private static Integer processSingleOperator(final String componentId, componentToSortedTasks.put(componentId, sortedTasks); Map outputStreams = new HashMap(); - for(Entry outStream : common.get_streams().entrySet()) { + for (Entry outStream : common.get_streams().entrySet()) { outputStreams.put(outStream.getKey(), new Fields(outStream.getValue().get_output_fields())); } componentToStreamToFields.put(componentId, outputStreams); diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java index ddbeaff9645677..d035bb2784d7b5 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkOutputFieldsDeclarerTest.java @@ -14,17 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.api; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.storm.util.AbstractTest; + +import org.apache.storm.tuple.Fields; +import org.apache.storm.utils.Utils; import org.junit.Assert; import org.junit.Test; import java.util.LinkedList; +/** + * Tests for the FlinkOutputFieldsDeclarer. + */ public class FlinkOutputFieldsDeclarerTest extends AbstractTest { @Test diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java index 0ec0179ebefe9c..aaecc061b18d18 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/FlinkTopologyTest.java @@ -14,17 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.api; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; import org.apache.flink.storm.util.TestDummyBolt; import org.apache.flink.storm.util.TestDummySpout; import org.apache.flink.storm.util.TestSink; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.tuple.Fields; import org.junit.Assert; import org.junit.Test; +/** + * Tests for the FlinkTopology. + */ public class FlinkTopologyTest { @Test @@ -68,7 +72,7 @@ public void testFieldsGroupingOnMultipleSpoutOutputStreams() { builder.setSpout("spout", new TestDummySpout()); builder.setBolt("sink", new TestSink()).fieldsGrouping("spout", - TestDummySpout.spoutStreamId, new Fields("id")); + TestDummySpout.SPOUT_STREAM_ID, new Fields("id")); FlinkTopology.createTopology(builder); } @@ -80,7 +84,7 @@ public void testFieldsGroupingOnMultipleBoltOutputStreams() { builder.setSpout("spout", new TestDummySpout()); builder.setBolt("bolt", new TestDummyBolt()).shuffleGrouping("spout"); builder.setBolt("sink", new TestSink()).fieldsGrouping("bolt", - TestDummyBolt.groupingStreamId, new Fields("id")); + TestDummyBolt.GROUPING_STREAM_ID, new Fields("id")); FlinkTopology.createTopology(builder); } diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java index 0f617fb766a866..001e9c4a542e50 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestBolt.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.api; import org.apache.storm.task.OutputCollector; @@ -24,6 +25,9 @@ import java.util.Map; +/** + * A no-op test implementation of a {@link IRichBolt}. + */ public class TestBolt implements IRichBolt { private static final long serialVersionUID = -667148827441397683L; diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java index 1b185a7ada99fc..3466ff40971e00 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/api/TestSpout.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.api; import org.apache.storm.spout.SpoutOutputCollector; @@ -23,6 +24,9 @@ import java.util.Map; +/** + * A no-op test implementation of a {@link IRichSpout}. + */ public class TestSpout implements IRichSpout { private static final long serialVersionUID = -4884029383198924007L; diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java index f51aba4bf216e4..ca0e06755e6d5a 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/AbstractTest.java @@ -23,6 +23,9 @@ import java.util.Random; +/** + * Abstract class for all tests that require a {@link Random} to be setup before each test. + */ public abstract class AbstractTest { private static final Logger LOG = LoggerFactory.getLogger(AbstractTest.class); diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java index 9a5b1cd7a02365..8b89c95161132c 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/FiniteTestSpout.java @@ -26,6 +26,9 @@ import java.util.Map; +/** + * Tests for the Finite. + */ public class FiniteTestSpout implements IRichSpout { private static final long serialVersionUID = 7992419478267824279L; diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java index 1eaed4aebf74ae..7263ce4a58ace9 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/NullTerminatingSpoutTest.java @@ -15,25 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.util; -import java.util.HashMap; -import java.util.Map; +package org.apache.flink.storm.util; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IRichSpout; import org.apache.storm.topology.OutputFieldsDeclarer; - import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Matchers.same; -import static org.mockito.Mockito.any; +/** + * Tests for the NullTerminatingSpout. + */ public class NullTerminatingSpoutTest { @Test @@ -43,7 +46,7 @@ public void testMethodCalls() { IRichSpout spoutMock = mock(IRichSpout.class); when(spoutMock.getComponentConfiguration()).thenReturn(compConfig); - Map conf = mock(Map.class); + Map conf = mock(Map.class); TopologyContext context = mock(TopologyContext.class); Object msgId = mock(Object.class); OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java index a5b96bde05f5f4..c150cc3c8f243b 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/SpoutOutputCollectorObserverTest.java @@ -15,15 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.storm.spout.SpoutOutputCollector; - import org.junit.Assert; import org.junit.Test; import static org.mockito.Mockito.mock; +/** + * Tests for the SpoutOutputCollectorObserver. + */ public class SpoutOutputCollectorObserverTest { @Test @@ -35,11 +38,11 @@ public void testFlag() { Assert.assertTrue(observer.emitted); observer.emitted = false; - observer.emit(null, (Object)null); + observer.emit(null, (Object) null); Assert.assertTrue(observer.emitted); observer.emitted = false; - observer.emit((String)null, null); + observer.emit((String) null, null); Assert.assertTrue(observer.emitted); observer.emitted = false; @@ -51,11 +54,11 @@ public void testFlag() { Assert.assertTrue(observer.emitted); observer.emitted = false; - observer.emitDirect(0, null, (Object)null); + observer.emitDirect(0, null, (Object) null); Assert.assertTrue(observer.emitted); observer.emitted = false; - observer.emitDirect(0, (String)null, null); + observer.emitDirect(0, (String) null, null); Assert.assertTrue(observer.emitted); observer.emitted = false; diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java index f73db2025f5e85..67d4a17e423223 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/StormStreamSelectorTest.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.junit.Assert; @@ -22,6 +23,9 @@ import java.util.Iterator; +/** + * Tests for the StormStreamSelector. + */ public class StormStreamSelectorTest { @Test diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java index 2ad8f2e4912364..277369285716b7 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummyBolt.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.storm.task.OutputCollector; @@ -26,11 +27,14 @@ import java.util.Map; +/** + * A test implementation of a {@link IRichBolt}. + */ public class TestDummyBolt implements IRichBolt { private static final long serialVersionUID = 6893611247443121322L; - public final static String shuffleStreamId = "shuffleStream"; - public final static String groupingStreamId = "groupingStream"; + public static final String SHUFFLE_STREAM_ID = "shuffleStream"; + public static final String GROUPING_STREAM_ID = "groupingStream"; private boolean emit = true; @SuppressWarnings("rawtypes") @@ -49,10 +53,10 @@ public void prepare(Map stormConf, TopologyContext context, OutputCollector coll @Override public void execute(Tuple input) { if (this.context.getThisTaskIndex() == 0) { - this.collector.emit(shuffleStreamId, input.getValues()); + this.collector.emit(SHUFFLE_STREAM_ID, input.getValues()); } if (this.emit) { - this.collector.emit(groupingStreamId, new Values("bolt", this.context)); + this.collector.emit(GROUPING_STREAM_ID, new Values("bolt", this.context)); this.emit = false; } } @@ -62,8 +66,8 @@ public void cleanup() {} @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(shuffleStreamId, new Fields("data")); - declarer.declareStream(groupingStreamId, new Fields("id", "data")); + declarer.declareStream(SHUFFLE_STREAM_ID, new Fields("data")); + declarer.declareStream(GROUPING_STREAM_ID, new Fields("id", "data")); } @Override diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java index 82506e40d17b5c..5ff8289dad638b 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestDummySpout.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.storm.spout.SpoutOutputCollector; @@ -26,10 +27,13 @@ import java.util.Map; +/** + * A test implementation of a {@link IRichSpout}. + */ public class TestDummySpout implements IRichSpout { private static final long serialVersionUID = -5190945609124603118L; - public final static String spoutStreamId = "spout-stream"; + public static final String SPOUT_STREAM_ID = "spout-stream"; private boolean emit = true; @SuppressWarnings("rawtypes") @@ -71,7 +75,7 @@ public void fail(Object msgId) {} @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declareStream(Utils.DEFAULT_STREAM_ID, new Fields("data")); - declarer.declareStream(spoutStreamId, new Fields("id", "data")); + declarer.declareStream(SPOUT_STREAM_ID, new Fields("id", "data")); } @Override diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java index 1f4da55f402c38..a3bb884788ae95 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/util/TestSink.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.storm.task.OutputCollector; @@ -26,23 +27,26 @@ import java.util.List; import java.util.Map; +/** + * A test implementation of a {@link IRichBolt} that stores incoming records in {@link #RESULT}. + */ public class TestSink implements IRichBolt { private static final long serialVersionUID = 4314871456719370877L; - public final static List result = new LinkedList(); + public static final List RESULT = new LinkedList(); @SuppressWarnings("rawtypes") @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { - result.add(context); + RESULT.add(context); } @Override public void execute(Tuple input) { if (input.size() == 1) { - result.add((TopologyContext) input.getValue(0)); + RESULT.add((TopologyContext) input.getValue(0)); } else { - result.add((TopologyContext) input.getValue(1)); + RESULT.add((TopologyContext) input.getValue(1)); } } diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java index 9e3165b31e286c..d48042b72449b8 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltCollectorTest.java @@ -17,11 +17,11 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.tuple.Values; - import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.storm.util.AbstractTest; import org.apache.flink.streaming.api.operators.Output; + +import org.apache.storm.tuple.Values; import org.junit.Assert; import org.junit.Test; @@ -32,6 +32,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +/** + * Tests for the BoltCollector. + */ public class BoltCollectorTest extends AbstractTest { @SuppressWarnings({ "rawtypes", "unchecked" }) diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java index 1f8f773a01c043..f518d178b5dfea 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java @@ -17,15 +17,6 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.MessageId; -import org.apache.storm.tuple.Values; -import org.apache.storm.utils.Utils; - import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.java.tuple.Tuple; @@ -46,6 +37,15 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTask; + +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.IRichBolt; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.MessageId; +import org.apache.storm.tuple.Values; +import org.apache.storm.utils.Utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -67,6 +67,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +/** + * Tests for the BoltWrapper. + */ @RunWith(PowerMockRunner.class) @PrepareForTest({StreamElementSerializer.class, WrapperSetupHelper.class, StreamRecord.class}) @PowerMockIgnore({"javax.management.*", "com.sun.jndi.*", "org.apache.log4j.*"}) diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java index 9a23b0f91e0fe1..0ee796b4db6ff6 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/FlinkTopologyContextTest.java @@ -17,6 +17,8 @@ package org.apache.flink.storm.wrappers; +import org.apache.flink.storm.util.AbstractTest; + import org.apache.storm.generated.Bolt; import org.apache.storm.generated.SpoutSpec; import org.apache.storm.generated.StateSpoutSpec; @@ -24,13 +26,11 @@ import org.apache.storm.metric.api.ICombiner; import org.apache.storm.metric.api.IMetric; import org.apache.storm.metric.api.IReducer; -import org.apache.flink.storm.util.AbstractTest; import org.junit.Test; import java.util.HashMap; - -/* +/** * FlinkTopologyContext.getSources(componentId) and FlinkTopologyContext.getTargets(componentId) are not tested here, * because those are tested in StormWrapperSetupHelperTest. */ diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java index 94a88fec846083..d6575d89d4723c 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SetupOutputFieldsDeclarerTest.java @@ -17,14 +17,18 @@ package org.apache.flink.storm.wrappers; +import org.apache.flink.storm.util.AbstractTest; + import org.apache.storm.tuple.Fields; import org.apache.storm.utils.Utils; -import org.apache.flink.storm.util.AbstractTest; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; +/** + * Tests for the SetupOutputFieldsDeclarer. + */ public class SetupOutputFieldsDeclarerTest extends AbstractTest { @Test diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java index eb91c63d80dbe7..b91871aa499c1e 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutCollectorTest.java @@ -17,11 +17,11 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.tuple.Values; - import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.storm.util.AbstractTest; import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; + +import org.apache.storm.tuple.Values; import org.junit.Assert; import org.junit.Test; @@ -31,6 +31,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +/** + * Tests for the SpoutCollector. + */ public class SpoutCollectorTest extends AbstractTest { @SuppressWarnings({ "rawtypes", "unchecked" }) diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java index 265e705b5f776f..e6d861bdeb0e84 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/SpoutWrapperTest.java @@ -17,10 +17,6 @@ package org.apache.flink.storm.wrappers; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.Fields; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.configuration.Configuration; @@ -31,6 +27,11 @@ import org.apache.flink.storm.util.TestDummySpout; import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; + +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.IRichSpout; +import org.apache.storm.tuple.Fields; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -51,6 +52,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +/** + * Tests for the SpoutWrapper. + */ @RunWith(PowerMockRunner.class) @PrepareForTest(WrapperSetupHelper.class) @PowerMockIgnore({"javax.management.*", "com.sun.jndi.*", "org.apache.log4j.*"}) diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java index 5e6c160649ffb6..2ff6c4598855d0 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/StormTupleTest.java @@ -17,15 +17,15 @@ package org.apache.flink.storm.wrappers; +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.api.java.tuple.Tuple5; import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.storm.util.AbstractTest; + import org.apache.storm.generated.GlobalStreamId; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.MessageId; import org.apache.storm.tuple.Values; - -import org.apache.flink.api.java.tuple.Tuple; -import org.apache.flink.api.java.tuple.Tuple5; -import org.apache.flink.storm.util.AbstractTest; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -35,6 +35,9 @@ import static org.mockito.Mockito.mock; +/** + * Tests for the StormTuple. + */ public class StormTupleTest extends AbstractTest { private static final String fieldName = "fieldName"; private static final String fieldNamePojo = "member"; @@ -638,8 +641,8 @@ private StormTuple testGetByField(int arity, int index, T value) tuple.setField(value, index); ArrayList attributeNames = new ArrayList(arity); - for(int i = 0; i < arity; ++i) { - if(i == index) { + for (int i = 0; i < arity; ++i) { + if (i == index) { attributeNames.add(fieldName); } else { attributeNames.add("" + i); @@ -685,7 +688,7 @@ public void testGetMessageId() { Assert.assertSame(messageId, stormTuple.getMessageId()); } - public static class TestPojoMember { + private static class TestPojoMember { public T member; public TestPojoMember(T value) { @@ -693,7 +696,7 @@ public TestPojoMember(T value) { } } - public static class TestPojoGetter { + private static class TestPojoGetter { private T member; public TestPojoGetter(T value) { diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java index 5f38705b0d19d6..3118d6bd6a506e 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupHelperTest.java @@ -18,6 +18,7 @@ package org.apache.flink.storm.wrappers; import org.apache.flink.storm.util.AbstractTest; + import org.apache.storm.topology.IComponent; import org.apache.storm.topology.IRichBolt; import org.apache.storm.topology.IRichSpout; @@ -37,6 +38,9 @@ import static java.util.Collections.singleton; import static org.mockito.Mockito.mock; +/** + * Tests for the WrapperSetupHelper. + */ @RunWith(PowerMockRunner.class) @PrepareForTest(WrapperSetupHelper.class) @PowerMockIgnore({"javax.*", "org.apache.log4j.*"}) diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java index 00173dfb4545a4..2b0b2753f2c2df 100644 --- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java +++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/WrapperSetupInLocalClusterTest.java @@ -23,6 +23,7 @@ import org.apache.flink.storm.util.TestDummySpout; import org.apache.flink.storm.util.TestSink; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; + import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.generated.ComponentCommon; @@ -36,11 +37,18 @@ import org.junit.Assert; import org.junit.Test; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Tests for the setup of wrappers in a local cluster. + */ public class WrapperSetupInLocalClusterTest extends AbstractTest { @Test @@ -73,17 +81,17 @@ public void testCreateTopologyContext() { builder.setBolt("bolt1", (IRichBolt) operators.get("bolt1"), dops.get("bolt1")).shuffleGrouping("spout1"); builder.setBolt("bolt2", (IRichBolt) operators.get("bolt2"), dops.get("bolt2")).allGrouping("spout2"); builder.setBolt("sink", (IRichBolt) operators.get("sink"), dops.get("sink")) - .shuffleGrouping("bolt1", TestDummyBolt.groupingStreamId) - .shuffleGrouping("bolt1", TestDummyBolt.shuffleStreamId) - .shuffleGrouping("bolt2", TestDummyBolt.groupingStreamId) - .shuffleGrouping("bolt2", TestDummyBolt.shuffleStreamId); + .shuffleGrouping("bolt1", TestDummyBolt.GROUPING_STREAM_ID) + .shuffleGrouping("bolt1", TestDummyBolt.SHUFFLE_STREAM_ID) + .shuffleGrouping("bolt2", TestDummyBolt.GROUPING_STREAM_ID) + .shuffleGrouping("bolt2", TestDummyBolt.SHUFFLE_STREAM_ID); LocalCluster cluster = new LocalCluster(); Config c = new Config(); c.setNumAckers(0); cluster.submitTopology("test", c, builder.createTopology()); - while (TestSink.result.size() != 8) { + while (TestSink.RESULT.size() != 8) { Utils.sleep(100); } cluster.shutdown(); @@ -92,7 +100,7 @@ public void testCreateTopologyContext() { Set taskIds = new HashSet(); - for (TopologyContext expectedContext : TestSink.result) { + for (TopologyContext expectedContext : TestSink.RESULT) { final String thisComponentId = expectedContext.getThisComponentId(); int index = taskCounter.get(thisComponentId); @@ -162,14 +170,14 @@ public void testCreateTopologyContext() { List possibleTasks = expectedContext.getComponentTasks(componentId); List tasks = topologyContext.getComponentTasks(componentId); - Iterator p_it = possibleTasks.iterator(); - Iterator t_it = tasks.iterator(); - while(p_it.hasNext()) { - Assert.assertTrue(t_it.hasNext()); - Assert.assertNull(taskToComponents.put(p_it.next(), componentId)); - Assert.assertTrue(allTaskIds.add(t_it.next())); + Iterator pIt = possibleTasks.iterator(); + Iterator tIt = tasks.iterator(); + while (pIt.hasNext()) { + Assert.assertTrue(tIt.hasNext()); + Assert.assertNull(taskToComponents.put(pIt.next(), componentId)); + Assert.assertTrue(allTaskIds.add(tIt.next())); } - Assert.assertFalse(t_it.hasNext()); + Assert.assertFalse(tIt.hasNext()); } Assert.assertEquals(taskToComponents, expectedContext.getTaskToComponent()); diff --git a/flink-contrib/flink-storm/src/test/resources/log4j-test.properties b/flink-contrib/flink-storm/src/test/resources/log4j-test.properties index 0b686e543bb236..881dc0609b8fdb 100644 --- a/flink-contrib/flink-storm/src/test/resources/log4j-test.properties +++ b/flink-contrib/flink-storm/src/test/resources/log4j-test.properties @@ -24,4 +24,4 @@ log4j.appender.A1=org.apache.log4j.ConsoleAppender # A1 uses PatternLayout. log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n From 5ea2240f0a0dd9e447abcb4493ecb7f1376505e5 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 23 May 2017 23:47:00 +0200 Subject: [PATCH 4/5] [FLINK-6695] Activate strict checkstyle for flink-storm-examples --- flink-contrib/flink-storm-examples/pom.xml | 35 +++++++++++++++++++ .../storm/exclamation/ExclamationLocal.java | 25 ++++++------- .../exclamation/ExclamationTopology.java | 25 ++++++------- .../exclamation/ExclamationWithBolt.java | 15 ++++---- .../exclamation/ExclamationWithSpout.java | 15 ++++---- .../operators/ExclamationBolt.java | 8 +++-- .../flink/storm/join/SingleJoinExample.java | 17 ++++----- .../flink/storm/print/PrintSampleStream.java | 14 ++++---- .../flink/storm/split/SpoutSplitExample.java | 11 +++--- .../storm/split/operators/RandomSpout.java | 10 ++++-- .../split/operators/VerifyAndEnrichBolt.java | 9 +++-- .../flink/storm/util/AbstractBoltSink.java | 4 +-- .../flink/storm/util/AbstractLineSpout.java | 2 +- .../apache/flink/storm/util/FileSpout.java | 6 ++-- .../flink/storm/util/FiniteFileSpout.java | 1 + .../flink/storm/util/OutputFormatter.java | 8 +++-- .../storm/util/SimpleOutputFormatter.java | 6 +++- .../storm/util/TupleOutputFormatter.java | 4 +++ .../wordcount/BoltTokenizerWordCount.java | 15 ++++---- .../wordcount/BoltTokenizerWordCountPojo.java | 15 ++++---- .../BoltTokenizerWordCountWithNames.java | 19 +++++----- .../storm/wordcount/SpoutSourceWordCount.java | 18 +++++----- .../flink/storm/wordcount/WordCountLocal.java | 26 +++++++------- .../storm/wordcount/WordCountLocalByName.java | 26 +++++++------- .../wordcount/WordCountRemoteByClient.java | 28 +++++++-------- .../wordcount/WordCountRemoteBySubmitter.java | 28 +++++++-------- .../storm/wordcount/WordCountTopology.java | 30 ++++++++-------- .../wordcount/operators/BoltCounter.java | 4 +-- .../operators/BoltCounterByName.java | 4 +-- .../wordcount/operators/BoltTokenizer.java | 4 +-- .../operators/BoltTokenizerByName.java | 4 +-- .../operators/WordCountDataPojos.java | 12 +++++-- .../operators/WordCountDataTuple.java | 5 ++- .../operators/WordCountInMemorySpout.java | 5 +-- .../ExclamationWithBoltITCase.java | 3 ++ .../ExclamationWithSpoutITCase.java | 3 ++ .../StormExclamationLocalITCase.java | 3 ++ .../exclamation/util/ExclamationData.java | 3 ++ .../flink/storm/join/SingleJoinITCase.java | 8 +++-- .../apache/flink/storm/split/SplitBolt.java | 8 +++-- .../flink/storm/split/SplitBoltTopology.java | 19 ++++++---- .../apache/flink/storm/split/SplitITCase.java | 11 ++++-- .../flink/storm/split/SplitSpoutTopology.java | 18 ++++++---- .../storm/split/SplitStreamBoltLocal.java | 11 ++++-- .../storm/split/SplitStreamSpoutLocal.java | 11 ++++-- .../tests/StormFieldsGroupingITCase.java | 24 +++++++------ .../storm/tests/StormMetaDataITCase.java | 18 ++++++---- .../flink/storm/tests/StormUnionITCase.java | 22 +++++++----- .../tests/operators/FiniteRandomSpout.java | 11 ++++-- .../storm/tests/operators/MergerBolt.java | 8 +++-- .../storm/tests/operators/MetaDataSpout.java | 8 +++-- .../storm/tests/operators/TaskIdBolt.java | 5 +-- .../tests/operators/VerifyMetaDataBolt.java | 8 +++-- .../BoltTokenizerWordCountITCase.java | 3 ++ .../BoltTokenizerWordCountPojoITCase.java | 3 ++ ...BoltTokenizerWordCountWithNamesITCase.java | 3 ++ .../wordcount/SpoutSourceWordCountITCase.java | 3 ++ .../storm/wordcount/WordCountLocalITCase.java | 3 ++ .../wordcount/WordCountLocalNamedITCase.java | 4 ++- 59 files changed, 425 insertions(+), 254 deletions(-) diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml index 6ef0f7b653bae9..0296ff3368878c 100644 --- a/flink-contrib/flink-storm-examples/pom.xml +++ b/flink-contrib/flink-storm-examples/pom.xml @@ -384,6 +384,41 @@ under the License. + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + /tools/maven/strict-checkstyle.xml + /tools/maven/suppressions.xml + true + true + true + + + + + test-compile + + check + + + + diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java index c37ae65f7a57c9..6108f791a9ab45 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationLocal.java @@ -17,34 +17,35 @@ package org.apache.flink.storm.exclamation; -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; import org.apache.flink.storm.exclamation.operators.ExclamationBolt; +import org.apache.storm.Config; +import org.apache.storm.topology.TopologyBuilder; + /** * Implements the "Exclamation" program that attaches five exclamation mark to every line of a text files in a streaming * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology} and submitted to * Flink for execution in the same way as to a Storm {@link org.apache.storm.LocalCluster}. - *

    - * This example shows how to run program directly within Java, thus it cannot be used to submit a + * + *

    This example shows how to run program directly within Java, thus it cannot be used to submit a * {@link org.apache.storm.generated.StormTopology} via Flink command line clients (ie, bin/flink). - *

    - * The input is a plain text file with lines separated by newline characters. - *

    - * Usage: ExclamationLocal <text path> <result path>
    + * + *

    The input is a plain text file with lines separated by newline characters. + * + *

    Usage: ExclamationLocal <text path> <result path>
    * If no parameters are provided, the program is run with default data from * {@link org.apache.flink.examples.java.wordcount.util.WordCountData}. - *

    - * This example shows how to: + * + *

    This example shows how to: *

      *
    • run a regular Storm program locally on Flink
    • *
    */ public class ExclamationLocal { - public final static String topologyId = "Streaming Exclamation"; + public static final String TOPOLOGY_ID = "Streaming Exclamation"; // ************************************************************************* // PROGRAM @@ -65,7 +66,7 @@ public static void main(final String[] args) throws Exception { conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true); // only required to stabilize integration test final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); - cluster.submitTopology(topologyId, conf, FlinkTopology.createTopology(builder)); + cluster.submitTopology(TOPOLOGY_ID, conf, FlinkTopology.createTopology(builder)); cluster.shutdown(); } diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java index 0144acbfa62bf2..51edd1f99fdf84 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationTopology.java @@ -17,7 +17,6 @@ package org.apache.flink.storm.exclamation; -import org.apache.storm.topology.TopologyBuilder; import org.apache.flink.examples.java.wordcount.util.WordCountData; import org.apache.flink.storm.exclamation.operators.ExclamationBolt; import org.apache.flink.storm.util.BoltFileSink; @@ -27,17 +26,19 @@ import org.apache.flink.storm.util.OutputFormatter; import org.apache.flink.storm.util.SimpleOutputFormatter; +import org.apache.storm.topology.TopologyBuilder; + /** * Implements the "Exclamation" program that attaches two exclamation marks to every line of a text files in a streaming * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology}. - *

    - * The input is a plain text file with lines separated by newline characters. - *

    - * Usage: Exclamation[Local|RemoteByClient|RemoteBySubmitter] <text path> + * + *

    The input is a plain text file with lines separated by newline characters. + * + *

    Usage: Exclamation[Local|RemoteByClient|RemoteBySubmitter] <text path> * <result path>
    * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

    - * This example shows how to: + * + *

    This example shows how to: *

      *
    • construct a regular Storm topology as Flink program
    • *
    • make use of the FiniteSpout interface
    • @@ -45,11 +46,11 @@ */ public class ExclamationTopology { - public final static String spoutId = "source"; - public final static String firstBoltId = "exclamation1"; - public final static String secondBoltId = "exclamation2"; - public final static String sinkId = "sink"; - private final static OutputFormatter formatter = new SimpleOutputFormatter(); + private static final String spoutId = "source"; + private static final String firstBoltId = "exclamation1"; + private static final String secondBoltId = "exclamation2"; + private static final String sinkId = "sink"; + private static final OutputFormatter formatter = new SimpleOutputFormatter(); public static TopologyBuilder buildTopology() { final TopologyBuilder builder = new TopologyBuilder(); diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java index 5a791198382ac3..a838e6901cf16e 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithBolt.java @@ -18,7 +18,6 @@ package org.apache.flink.storm.exclamation; -import org.apache.storm.utils.Utils; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.examples.java.wordcount.util.WordCountData; @@ -28,17 +27,19 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.storm.utils.Utils; + /** * Implements the "Exclamation" program that attaches 3+x exclamation marks to every line of a text files in a streaming * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology}. - *

      - * The input is a plain text file with lines separated by newline characters. - *

      - * Usage: + * + *

      The input is a plain text file with lines separated by newline characters. + * + *

      Usage: * ExclamationWithmBolt <text path> <result path> <number of exclamation marks>
      * If no parameters are provided, the program is run with default data from {@link WordCountData} with x=2. - *

      - * This example shows how to: + * + *

      This example shows how to: *

        *
      • use a Bolt within a Flink Streaming program
      • *
      • how to configure a Bolt using StormConfig
      • diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java index 237f1d4e4c436f..b165f00ee7da69 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/ExclamationWithSpout.java @@ -18,7 +18,6 @@ package org.apache.flink.storm.exclamation; -import org.apache.storm.utils.Utils; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.examples.java.wordcount.util.WordCountData; @@ -29,16 +28,18 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.storm.utils.Utils; + /** * Implements the "Exclamation" program that attaches six exclamation marks to every line of a text files in a streaming * fashion. The program is constructed as a regular {@link org.apache.storm.generated.StormTopology}. - *

        - * The input is a plain text file with lines separated by newline characters. - *

        - * Usage: ExclamationWithSpout <text path> <result path>
        + * + *

        The input is a plain text file with lines separated by newline characters. + * + *

        Usage: ExclamationWithSpout <text path> <result path>
        * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

        - * This example shows how to: + * + *

        This example shows how to: *

          *
        • use a Storm spout within a Flink Streaming program
        • *
        • make use of the FiniteSpout interface
        • diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java index 77a91d2e2a3c9b..8872acdd9cb699 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/exclamation/operators/ExclamationBolt.java @@ -28,10 +28,14 @@ import java.util.Map; +/** + * A Bolt implementation that appends exclamation marks to incoming tuples. The number of added exclamation marks can + * be controlled by setting exclamation.count. + */ public class ExclamationBolt implements IRichBolt { - private final static long serialVersionUID = -6364882114201311380L; + private static final long serialVersionUID = -6364882114201311380L; - public final static String EXCLAMATION_COUNT = "exclamation.count"; + public static final String EXCLAMATION_COUNT = "exclamation.count"; private OutputCollector collector; private String exclamation; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java index 41ea4cbdbfa7fb..b2ad05f1bb5a76 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/join/SingleJoinExample.java @@ -15,13 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.join; -import org.apache.storm.Config; -import org.apache.storm.testing.FeederSpout; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; +package org.apache.flink.storm.join; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; @@ -29,10 +24,17 @@ import org.apache.flink.storm.util.NullTerminatingSpout; import org.apache.flink.storm.util.TupleOutputFormatter; +import org.apache.storm.Config; import org.apache.storm.starter.bolt.PrinterBolt; import org.apache.storm.starter.bolt.SingleJoinBolt; +import org.apache.storm.testing.FeederSpout; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; - +/** + * Implements a simple example where 2 input streams are being joined. + */ public class SingleJoinExample { public static void main(String[] args) throws Exception { @@ -79,7 +81,6 @@ public static void main(String[] args) throws Exception { ageSpout.feed(new Values(i, i + 20)); } - final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); cluster.submitTopology("joinTopology", conf, FlinkTopology.createTopology(builder)); cluster.shutdown(); diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java index da2e641f9c1862..6157e2cf495d87 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/print/PrintSampleStream.java @@ -18,13 +18,14 @@ package org.apache.flink.storm.print; -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; + +import org.apache.storm.Config; import org.apache.storm.starter.bolt.PrinterBolt; import org.apache.storm.starter.spout.TwitterSampleSpout; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.utils.Utils; import java.util.Arrays; @@ -32,15 +33,15 @@ * Prints incoming tweets. Tweets can be filtered by keywords. */ public class PrintSampleStream { - + public static void main(String[] args) throws Exception { - + if (args.length < 4) { System.err.println( "Usage: PrintSampleStream "); return; } - + String consumerKey = args[0]; String consumerSecret = args[1]; String accessToken = args[2]; @@ -56,7 +57,6 @@ public static void main(String[] args) throws Exception { builder.setBolt("print", new PrinterBolt()) .shuffleGrouping("twitter"); - Config conf = new Config(); final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster(); diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java index 02131fc641a1b4..c5bb5c398c93e1 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/SpoutSplitExample.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.split; import org.apache.flink.api.common.functions.MapFunction; @@ -33,15 +34,15 @@ /** * Implements a simple example with two declared output streams for the embedded spout. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • handle multiple output stream of a spout
          • *
          • accessing each stream by .split(...) and .select(...)
          • *
          • strip wrapper data type SplitStreamType for further processing in Flink
          • *
          - *

          - * This example would work the same way for multiple bolt output streams. + * + *

          This example would work the same way for multiple bolt output streams. */ public class SpoutSplitExample { @@ -94,7 +95,7 @@ public static void main(final String[] args) throws Exception { /** * Same as {@link VerifyAndEnrichBolt}. */ - public final static class Enrich implements MapFunction> { + public static final class Enrich implements MapFunction> { private static final long serialVersionUID = 5213888269197438892L; private final Tuple2 out; private final boolean isEven; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java index 5fbe0a78498f70..afec47f0fe187e 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/RandomSpout.java @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.split.operators; -import java.util.Map; -import java.util.Random; +package org.apache.flink.storm.split.operators; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; @@ -27,6 +25,12 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; +import java.util.Map; +import java.util.Random; + +/** + * A Spout implementation that emits random numbers, optionally splitting them into odd/even streams. + */ public class RandomSpout extends BaseRichSpout { private static final long serialVersionUID = -3978554318742509334L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java index 1ad9a6c2bd3c90..a39ec9c34f515c 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/split/operators/VerifyAndEnrichBolt.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.split.operators; -import java.util.Map; +package org.apache.flink.storm.split.operators; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -27,6 +26,12 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; +import java.util.Map; + +/** + * Verifies that incoming numbers are either even or odd, controlled by the constructor argument. Emitted tuples are + * enriched with a new string field containing either "even" or "odd", based on the number's parity. + */ public class VerifyAndEnrichBolt extends BaseRichBolt { private static final long serialVersionUID = -7277395570966328721L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java index 2cb346ae7c126e..5ae8cfbe205d0a 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractBoltSink.java @@ -50,7 +50,7 @@ public final void prepare(final Map stormConf, final TopologyContext context, } } - protected abstract void prepareSimple(final Map stormConf, final TopologyContext context); + protected abstract void prepareSimple(Map stormConf, TopologyContext context); @Override public final void execute(final Tuple input) { @@ -60,7 +60,7 @@ public final void execute(final Tuple input) { this.writeExternal(this.lineBuilder.toString()); } - protected abstract void writeExternal(final String line); + protected abstract void writeExternal(String line); @Override public void cleanup() {/* nothing to do */} diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java index 29df23ece4e32f..caefd567c703fe 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/AbstractLineSpout.java @@ -32,7 +32,7 @@ public abstract class AbstractLineSpout implements IRichSpout { private static final long serialVersionUID = 8876828403487806771L; - public final static String ATTRIBUTE_LINE = "line"; + public static final String ATTRIBUTE_LINE = "line"; protected SpoutOutputCollector collector; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java index 0a295e771790c2..0533b0908ed622 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FileSpout.java @@ -33,7 +33,7 @@ public class FileSpout extends AbstractLineSpout { private static final long serialVersionUID = -6996907090003590436L; - public final static String INPUT_FILE_PATH = "input.path"; + public static final String INPUT_FILE_PATH = "input.path"; protected String path = null; protected BufferedReader reader; @@ -50,8 +50,8 @@ public void open(final Map conf, final TopologyContext context, final SpoutOutpu super.open(conf, context, collector); Object configuredPath = conf.get(INPUT_FILE_PATH); - if(configuredPath != null) { - this.path = (String)configuredPath; + if (configuredPath != null) { + this.path = (String) configuredPath; } try { diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java index 48349c2803a7a0..e4f39ab13e15b1 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/FiniteFileSpout.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.util; import org.apache.storm.spout.SpoutOutputCollector; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java index fe28afc87262c2..a0f933f1aacb8a 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/OutputFormatter.java @@ -22,16 +22,20 @@ import java.io.Serializable; +/** + * Interface that is used to convert Storm {@link Tuple Tuples} to a string before writing them out to a file or to the + * console. + */ public interface OutputFormatter extends Serializable { /** * Converts a Storm {@link Tuple} to a string. This method is used for formatting the output tuples before writing * them out to a file or to the console. - * + * * @param input * The tuple to be formatted * @return The string result of the formatting */ - public String format(Tuple input); + String format(Tuple input); } diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java index 323fb538842294..bf30cd21c0b6ea 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/SimpleOutputFormatter.java @@ -20,13 +20,17 @@ import org.apache.storm.tuple.Tuple; +/** + * Simple {@link OutputFormatter} implementation to convert {@link Tuple Tuples} with a size of 1 by returning the + * result of {@link Object#toString()} for the first field. + */ public class SimpleOutputFormatter implements OutputFormatter { private static final long serialVersionUID = 6349573860144270338L; /** * Converts a Storm {@link Tuple} with 1 field to a string by retrieving the value of that field. This method is * used for formatting raw outputs wrapped in tuples, before writing them out to a file or to the console. - * + * * @param input * The tuple to be formatted * @return The string result of the formatting diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java index 11d23cddaccdb6..42189a70f200f4 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/util/TupleOutputFormatter.java @@ -20,6 +20,10 @@ import org.apache.storm.tuple.Tuple; +/** + * {@link OutputFormatter} implementation that converts {@link Tuple Tuples} of arbitrary size to a string. For a given + * tuple the output is (field1,field2,...,fieldX). + */ public class TupleOutputFormatter implements OutputFormatter { private static final long serialVersionUID = -599665757723851761L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java index 4620d9d0cf1670..6f7addf6675cd4 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCount.java @@ -17,7 +17,6 @@ package org.apache.flink.storm.wordcount; -import org.apache.storm.topology.IRichBolt; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.examples.java.wordcount.util.WordCountData; @@ -26,16 +25,18 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.storm.topology.IRichBolt; + /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The tokenizer step is performed by a {@link IRichBolt Bolt}. - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCount <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCount <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • use a Bolt within a Flink Streaming program.
          • *
          diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java index eefbf78481ac02..125a044d18a3d1 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojo.java @@ -17,7 +17,6 @@ package org.apache.flink.storm.wordcount; -import org.apache.storm.topology.IRichBolt; import org.apache.flink.api.java.io.CsvInputFormat; import org.apache.flink.api.java.io.PojoCsvInputFormat; import org.apache.flink.api.java.tuple.Tuple2; @@ -32,17 +31,19 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.storm.topology.IRichBolt; + /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The tokenizer step is performed by a {@link IRichBolt Bolt}. In contrast to {@link BoltTokenizerWordCount} * the tokenizer's input is a POJO type and the single field is accessed by name. - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCount <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCount <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • how to access attributes by name within a Bolt for POJO type input streams *
          diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java index 98f7f96324b1ec..f469babcf8a390 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNames.java @@ -17,8 +17,6 @@ package org.apache.flink.storm.wordcount; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.tuple.Fields; import org.apache.flink.api.java.io.CsvInputFormat; import org.apache.flink.api.java.io.TupleCsvInputFormat; import org.apache.flink.api.java.tuple.Tuple; @@ -34,17 +32,20 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.storm.topology.IRichBolt; +import org.apache.storm.tuple.Fields; + /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The tokenizer step is performed by a {@link IRichBolt Bolt}. In contrast to {@link BoltTokenizerWordCount} * the tokenizer's input is a {@link Tuple} type and the single field is accessed by name. - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCount <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCount <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • how to access attributes by name within a Bolt for {@link Tuple} type input streams *
          @@ -120,7 +121,7 @@ private static boolean parseParameters(final String[] args) { private static DataStream> getTextDataStream(final StreamExecutionEnvironment env) { if (fileOutput) { // read the text file from given input path - TupleTypeInfo> sourceType = (TupleTypeInfo>)TypeExtractor + TupleTypeInfo> sourceType = (TupleTypeInfo>) TypeExtractor .getForObject(new Tuple1("")); return env.createInput(new TupleCsvInputFormat>(new Path( textPath), CsvInputFormat.DEFAULT_LINE_DELIMITER, diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java index 683a3b5480bfce..f0cfd7a1a14c79 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/SpoutSourceWordCount.java @@ -17,9 +17,6 @@ package org.apache.flink.storm.wordcount; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.utils.Utils; - import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; @@ -31,16 +28,19 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Collector; +import org.apache.storm.topology.IRichSpout; +import org.apache.storm.utils.Utils; + /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The used data source is a {@link IRichSpout Spout}. - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCount <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCount <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • use a Spout within a Flink Streaming program.
          • *
          diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java index ee880badf9e563..82c8ae321bbee2 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocal.java @@ -17,35 +17,35 @@ package org.apache.flink.storm.wordcount; +import org.apache.flink.examples.java.wordcount.util.WordCountData; +import org.apache.flink.storm.api.FlinkLocalCluster; +import org.apache.flink.storm.api.FlinkTopology; + import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.generated.StormTopology; import org.apache.storm.topology.TopologyBuilder; -import org.apache.flink.examples.java.wordcount.util.WordCountData; -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; - /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the * same way as to a Storm {@link LocalCluster}. - *

          - * This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology} + * + *

          This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology} * via Flink command line clients (ie, bin/flink). - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCount <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCount <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • run a regular Storm program locally on Flink
          • *
          */ public class WordCountLocal { - public final static String topologyId = "Storm WordCount"; + private static final String topologyId = "Storm WordCount"; // ************************************************************************* // PROGRAM diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java index ab423cf29e6631..b960b795c2eac4 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountLocalByName.java @@ -17,36 +17,36 @@ package org.apache.flink.storm.wordcount; +import org.apache.flink.examples.java.wordcount.util.WordCountData; +import org.apache.flink.storm.api.FlinkLocalCluster; +import org.apache.flink.storm.api.FlinkTopology; + import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.generated.StormTopology; import org.apache.storm.topology.TopologyBuilder; -import org.apache.flink.examples.java.wordcount.util.WordCountData; -import org.apache.flink.storm.api.FlinkLocalCluster; -import org.apache.flink.storm.api.FlinkTopology; - /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the * same way as to a Storm {@link LocalCluster}. In contrast to {@link WordCountLocal} all bolts access the field of * input tuples by name instead of index. - *

          - * This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology} + * + *

          This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology} * via Flink command line clients (ie, bin/flink). - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCountLocalByName <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCountLocalByName <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • run a regular Storm program locally on Flink *
          */ public class WordCountLocalByName { - public final static String topologyId = "Storm WordCountName"; + private static final String topologyId = "Storm WordCountName"; // ************************************************************************* // PROGRAM diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java index 5c99f933c71ff0..8dff6d7273da8d 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteByClient.java @@ -17,6 +17,10 @@ package org.apache.flink.storm.wordcount; +import org.apache.flink.examples.java.wordcount.util.WordCountData; +import org.apache.flink.storm.api.FlinkClient; +import org.apache.flink.storm.api.FlinkTopology; + import org.apache.storm.Config; import org.apache.storm.generated.AlreadyAliveException; import org.apache.storm.generated.InvalidTopologyException; @@ -26,31 +30,27 @@ import org.apache.storm.utils.NimbusClient; import org.apache.storm.utils.Utils; -import org.apache.flink.examples.java.wordcount.util.WordCountData; -import org.apache.flink.storm.api.FlinkClient; -import org.apache.flink.storm.api.FlinkTopology; - /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the * same way as to a Storm cluster similar to {@link NimbusClient}. The Flink cluster can be local or remote. - *

          - * This example shows how to submit the program via Java, thus it cannot be used to submit a {@link StormTopology} via + * + *

          This example shows how to submit the program via Java, thus it cannot be used to submit a {@link StormTopology} via * Flink command line clients (ie, bin/flink). - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCountRemoteByClient <text path> <result path>
          + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCountRemoteByClient <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • submit a regular Storm program to a local or remote Flink cluster.
          • *
          */ public class WordCountRemoteByClient { - public final static String topologyId = "Storm WordCount"; - private final static String uploadedJarLocation = "WordCount-StormTopology.jar"; + private static final String topologyId = "Storm WordCount"; + private static final String uploadedJarLocation = "WordCount-StormTopology.jar"; // ************************************************************************* // PROGRAM diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java index 08ba52aeccd26e..745ec85cbbc89b 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountRemoteBySubmitter.java @@ -17,35 +17,35 @@ package org.apache.flink.storm.wordcount; -import org.apache.storm.Config; -import org.apache.storm.StormSubmitter; -import org.apache.storm.generated.StormTopology; - -import org.apache.storm.topology.TopologyBuilder; import org.apache.flink.examples.java.wordcount.util.WordCountData; import org.apache.flink.storm.api.FlinkClient; import org.apache.flink.storm.api.FlinkSubmitter; import org.apache.flink.storm.api.FlinkTopology; +import org.apache.storm.Config; +import org.apache.storm.StormSubmitter; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.topology.TopologyBuilder; + /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the * same way as to a Storm cluster similar to {@link StormSubmitter}. The Flink cluster can be local or remote. - *

          - * This example shows how to submit the program via Java as well as Flink's command line client (ie, bin/flink). - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: WordCountRemoteBySubmitter <text path> <result path>
          + * + *

          This example shows how to submit the program via Java as well as Flink's command line client (ie, bin/flink). + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: WordCountRemoteBySubmitter <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • submit a regular Storm program to a local or remote Flink cluster.
          • *
          */ public class WordCountRemoteBySubmitter { - public final static String topologyId = "Storm WordCount"; + private static final String topologyId = "Storm WordCount"; // ************************************************************************* // PROGRAM diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java index 8f855b5ce86102..8627145b012576 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/WordCountTopology.java @@ -17,10 +17,6 @@ package org.apache.flink.storm.wordcount; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; - import org.apache.flink.examples.java.wordcount.util.WordCountData; import org.apache.flink.storm.util.BoltFileSink; import org.apache.flink.storm.util.BoltPrintSink; @@ -34,27 +30,31 @@ import org.apache.flink.storm.wordcount.operators.WordCountFileSpout; import org.apache.flink.storm.wordcount.operators.WordCountInMemorySpout; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.tuple.Fields; + /** * Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming * fashion. The program is constructed as a regular {@link StormTopology}. - *

          - * The input is a plain text file with lines separated by newline characters. - *

          - * Usage: + * + *

          The input is a plain text file with lines separated by newline characters. + * + *

          Usage: * WordCount[Local|LocalByName|RemoteByClient|RemoteBySubmitter] <text path> <result path>
          * If no parameters are provided, the program is run with default data from {@link WordCountData}. - *

          - * This example shows how to: + * + *

          This example shows how to: *

            *
          • how to construct a regular Storm topology as Flink program
          • *
          */ public class WordCountTopology { - public final static String spoutId = "source"; - public final static String tokenierzerId = "tokenizer"; - public final static String counterId = "counter"; - public final static String sinkId = "sink"; - private final static OutputFormatter formatter = new TupleOutputFormatter(); + private static final String spoutId = "source"; + private static final String tokenierzerId = "tokenizer"; + private static final String counterId = "counter"; + private static final String sinkId = "sink"; + private static final OutputFormatter formatter = new TupleOutputFormatter(); public static TopologyBuilder buildTopology() { return buildTopology(true); diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java index 4a00869c9f4e20..34fc703f6be555 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounter.java @@ -32,8 +32,8 @@ * Implements the word counter that counts the occurrence of each unique word. The bolt takes a pair (input tuple * schema: {@code }) and sums the given word count for each unique word (output tuple schema: * {@code } ). - *

          - * Same as {@link BoltCounterByName}, but accesses input attribute by index (instead of name). + * + *

          Same as {@link BoltCounterByName}, but accesses input attribute by index (instead of name). */ public class BoltCounter implements IRichBolt { private static final long serialVersionUID = 399619605462625934L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java index e3e0d58f9db191..cd53d505564377 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltCounterByName.java @@ -32,8 +32,8 @@ * Implements the word counter that counts the occurrence of each unique word. The bolt takes a pair (input tuple * schema: {@code }) and sums the given word count for each unique word (output tuple schema: * {@code } ). - *

          - * Same as {@link BoltCounter}, but accesses input attribute by name (instead of index). + * + *

          Same as {@link BoltCounter}, but accesses input attribute by name (instead of index). */ public class BoltCounterByName implements IRichBolt { private static final long serialVersionUID = 399619605462625934L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java index cedd90a5a44db8..41e8a8d5a5b98b 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizer.java @@ -31,8 +31,8 @@ * Implements the string tokenizer that splits sentences into words as a bolt. The bolt takes a line (input tuple * schema: {@code }) and splits it into multiple pairs in the form of "(word,1)" (output tuple schema: * {@code }). - *

          - * Same as {@link BoltTokenizerByName}, but accesses input attribute by index (instead of name). + * + *

          Same as {@link BoltTokenizerByName}, but accesses input attribute by index (instead of name). */ public final class BoltTokenizer implements IRichBolt { private static final long serialVersionUID = -8589620297208175149L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java index 258d4129d34a35..dff39eba1373f4 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/BoltTokenizerByName.java @@ -31,8 +31,8 @@ * Implements the string tokenizer that splits sentences into words as a bolt. The bolt takes a line (input tuple * schema: {@code }) and splits it into multiple pairs in the form of "(word,1)" (output tuple schema: * {@code }). - *

          - * Same as {@link BoltTokenizer}, but accesses input attribute by name (instead of index). + * + *

          Same as {@link BoltTokenizer}, but accesses input attribute by name (instead of index). */ public final class BoltTokenizerByName implements IRichBolt { private static final long serialVersionUID = -8589620297208175149L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java index 3a8fd3ad5716fc..d63974b94f75e0 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataPojos.java @@ -17,12 +17,15 @@ package org.apache.flink.storm.wordcount.operators; -import java.io.Serializable; - import org.apache.flink.examples.java.wordcount.util.WordCountData; +import java.io.Serializable; + +/** + * Input POJOs for WordCount programs. + */ public class WordCountDataPojos { - public static Sentence[] SENTENCES; + public static final Sentence[] SENTENCES; static { SENTENCES = new Sentence[WordCountData.WORDS.length]; @@ -31,6 +34,9 @@ public class WordCountDataPojos { } } + /** + * Simple POJO containing a string. + */ public static class Sentence implements Serializable { private static final long serialVersionUID = -7336372859203407522L; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java index 16e2ba00f77fbe..d01d9a25371a0a 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountDataTuple.java @@ -20,9 +20,12 @@ import org.apache.flink.api.java.tuple.Tuple1; import org.apache.flink.examples.java.wordcount.util.WordCountData; +/** + * Input tuples for WordCount programs. + */ @SuppressWarnings("unchecked") public class WordCountDataTuple { - public static Tuple1[] TUPLES; + public static final Tuple1[] TUPLES; static { TUPLES = new Tuple1[WordCountData.WORDS.length]; diff --git a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java index 7bf40c2fcadf13..be376a9f303745 100644 --- a/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java +++ b/flink-contrib/flink-storm-examples/src/main/java/org/apache/flink/storm/wordcount/operators/WordCountInMemorySpout.java @@ -17,11 +17,12 @@ package org.apache.flink.storm.wordcount.operators; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.tuple.Fields; import org.apache.flink.examples.java.wordcount.util.WordCountData; import org.apache.flink.storm.util.FiniteInMemorySpout; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.tuple.Fields; + /** * Implements a Spout that reads data from {@link WordCountData#WORDS}. */ diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java index 5a37572601e1e1..358919fea2d596 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithBoltITCase.java @@ -22,6 +22,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the ExclamationWithBolt example. + */ public class ExclamationWithBoltITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java index c2b0467c38d9d9..61310e897e85be 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/ExclamationWithSpoutITCase.java @@ -22,6 +22,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the ExclamantionWithSpout example. + */ public class ExclamationWithSpoutITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java index 049c88112576b3..bc09a3d4a50f76 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/StormExclamationLocalITCase.java @@ -22,6 +22,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the ExclamationLocal example. + */ public class StormExclamationLocalITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java index 3c435f982afda4..f7000097921ddc 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/exclamation/util/ExclamationData.java @@ -18,6 +18,9 @@ package org.apache.flink.storm.exclamation.util; +/** + * Expected output of Exclamation programs. + */ public class ExclamationData { public static final String TEXT_WITH_EXCLAMATIONS = diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java index b51db2c557c314..83531bab38a2e1 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/join/SingleJoinITCase.java @@ -18,12 +18,16 @@ package org.apache.flink.storm.join; -import com.google.common.base.Joiner; import org.apache.flink.streaming.util.StreamingProgramTestBase; +import com.google.common.base.Joiner; + +/** + * Test for the SingleJoin example. + */ public class SingleJoinITCase extends StreamingProgramTestBase { - protected static String expectedOutput[] = { + protected static String[] expectedOutput = { "(male,20)", "(female,21)", "(male,22)", diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java index 0fc1ba58ceeefd..90ee795317f81c 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBolt.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.split; -import java.util.Map; +package org.apache.flink.storm.split; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -27,6 +26,11 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; +import java.util.Map; + +/** + * A bolt for splitting an input stream containing numbers based on whether they are even or odd. + */ public class SplitBolt extends BaseRichBolt { private static final long serialVersionUID = -6627606934204267173L; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java index 04cfeedac4edeb..c00284035db08b 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitBoltTopology.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.split; -import org.apache.storm.topology.TopologyBuilder; import org.apache.flink.storm.split.operators.RandomSpout; import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; import org.apache.flink.storm.util.BoltFileSink; @@ -25,13 +25,18 @@ import org.apache.flink.storm.util.OutputFormatter; import org.apache.flink.storm.util.TupleOutputFormatter; +import org.apache.storm.topology.TopologyBuilder; + +/** + * A simple topology that splits a stream of numbers based on their parity, and verifies the result. + */ public class SplitBoltTopology { - public final static String spoutId = "randomSource"; - public final static String boltId = "splitBolt"; - public final static String evenVerifierId = "evenVerifier"; - public final static String oddVerifierId = "oddVerifier"; - public final static String sinkId = "sink"; - private final static OutputFormatter formatter = new TupleOutputFormatter(); + private static final String spoutId = "randomSource"; + private static final String boltId = "splitBolt"; + private static final String evenVerifierId = "evenVerifier"; + private static final String oddVerifierId = "oddVerifier"; + private static final String sinkId = "sink"; + private static final OutputFormatter formatter = new TupleOutputFormatter(); public static TopologyBuilder buildTopology() { final TopologyBuilder builder = new TopologyBuilder(); diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java index 4da9708a731e40..d53493cb46a7d2 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitITCase.java @@ -14,19 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.split; -import java.io.File; -import java.io.IOException; +package org.apache.flink.storm.split; import org.apache.flink.storm.split.SpoutSplitExample.Enrich; import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase; + import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; + +/** + * Tests for split examples. + */ public class SplitITCase extends StreamingMultipleProgramsTestBase { private String output; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java index 8671d2e91ccb9f..aa92a959995626 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitSpoutTopology.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.split; -import org.apache.storm.topology.TopologyBuilder; import org.apache.flink.storm.split.operators.RandomSpout; import org.apache.flink.storm.split.operators.VerifyAndEnrichBolt; import org.apache.flink.storm.util.BoltFileSink; @@ -25,12 +25,18 @@ import org.apache.flink.storm.util.OutputFormatter; import org.apache.flink.storm.util.TupleOutputFormatter; +import org.apache.storm.topology.TopologyBuilder; + +/** + * A simple topology similar to the {@link SplitBoltTopology}, except that the split streams are generated directly in + * a spout. + */ public class SplitSpoutTopology { - public final static String spoutId = "randomSplitSource"; - public final static String evenVerifierId = "evenVerifier"; - public final static String oddVerifierId = "oddVerifier"; - public final static String sinkId = "sink"; - private final static OutputFormatter formatter = new TupleOutputFormatter(); + private static final String spoutId = "randomSplitSource"; + private static final String evenVerifierId = "evenVerifier"; + private static final String oddVerifierId = "oddVerifier"; + private static final String sinkId = "sink"; + private static final OutputFormatter formatter = new TupleOutputFormatter(); public static TopologyBuilder buildTopology() { final TopologyBuilder builder = new TopologyBuilder(); diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java index 2cde11e953dd9f..55c3bd39fb7fca 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamBoltLocal.java @@ -14,15 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.split; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.utils.Utils; + +/** + * An example using the {@link SplitBoltTopology}. + */ public class SplitStreamBoltLocal { - public final static String topologyId = "Bolt split stream example"; + private static final String topologyId = "Bolt split stream example"; // ************************************************************************* // PROGRAM diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java index be880d01875881..da6e574131a742 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/split/SplitStreamSpoutLocal.java @@ -14,15 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.split; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.utils.Utils; + +/** + * An example using the {@link SplitSpoutTopology}. + */ public class SplitStreamSpoutLocal { - public final static String topologyId = "Spout split stream example"; + private static final String topologyId = "Spout split stream example"; // ************************************************************************* // PROGRAM diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java index 581f7c1feab371..c861c9e2587a14 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormFieldsGroupingITCase.java @@ -15,11 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.storm.tests; -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.tuple.Fields; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; import org.apache.flink.storm.tests.operators.FiniteRandomSpout; @@ -28,6 +26,10 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.util.MathUtils; + +import org.apache.storm.Config; +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.tuple.Fields; import org.junit.Assert; import java.util.ArrayList; @@ -41,10 +43,10 @@ */ public class StormFieldsGroupingITCase extends StreamingProgramTestBase { - private final static String topologyId = "FieldsGrouping Test"; - private final static String spoutId = "spout"; - private final static String boltId = "bolt"; - private final static String sinkId = "sink"; + private static final String topologyId = "FieldsGrouping Test"; + private static final String spoutId = "spout"; + private static final String boltId = "bolt"; + private static final String sinkId = "sink"; private String resultPath; @Override @@ -62,19 +64,19 @@ protected void postSubmit() throws Exception { readAllResultLines(actualResults, resultPath, new String[0], false); //remove potential operator id prefix - for(int i = 0; i < actualResults.size(); ++i) { + for (int i = 0; i < actualResults.size(); ++i) { String s = actualResults.get(i); - if(s.contains(">")) { + if (s.contains(">")) { s = s.substring(s.indexOf(">") + 2); actualResults.set(i, s); } } - Assert.assertEquals(expectedResults.size(),actualResults.size()); + Assert.assertEquals(expectedResults.size(), actualResults.size()); Collections.sort(actualResults); Collections.sort(expectedResults); System.out.println(actualResults); - for(int i=0; i< actualResults.size(); ++i) { + for (int i = 0; i < actualResults.size(); ++i) { //compare against actual results with removed prefex (as it depends e.g. on the hash function used) Assert.assertEquals(expectedResults.get(i), actualResults.get(i)); } diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java index b19e106328d81d..fe09dafd916a38 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormMetaDataITCase.java @@ -15,24 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests; -import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.utils.Utils; +package org.apache.flink.storm.tests; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; import org.apache.flink.storm.tests.operators.MetaDataSpout; import org.apache.flink.storm.tests.operators.VerifyMetaDataBolt; import org.apache.flink.streaming.util.StreamingProgramTestBase; + +import org.apache.storm.topology.TopologyBuilder; +import org.apache.storm.utils.Utils; import org.junit.Assert; +/** + * Test for meta data spouts/bolts. + */ public class StormMetaDataITCase extends StreamingProgramTestBase { - private final static String topologyId = "FieldsGrouping Test"; - private final static String spoutId = "spout"; - private final static String boltId1 = "bolt1"; - private final static String boltId2 = "bolt2"; + private static final String topologyId = "FieldsGrouping Test"; + private static final String spoutId = "spout"; + private static final String boltId1 = "bolt1"; + private static final String boltId2 = "bolt2"; @Override protected void testProgram() throws Exception { diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java index 452fef518daf5a..12e897acb896a2 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/StormUnionITCase.java @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests; -import org.apache.storm.Config; -import org.apache.storm.topology.TopologyBuilder; +package org.apache.flink.storm.tests; import org.apache.flink.storm.api.FlinkLocalCluster; import org.apache.flink.storm.api.FlinkTopology; @@ -27,6 +25,12 @@ import org.apache.flink.storm.util.BoltFileSink; import org.apache.flink.streaming.util.StreamingProgramTestBase; +import org.apache.storm.Config; +import org.apache.storm.topology.TopologyBuilder; + +/** + * Test for the {@link MergerBolt}. + */ public class StormUnionITCase extends StreamingProgramTestBase { private static final String RESULT = "-1154715079\n" + "-1155869325\n" + "-1155484576\n" @@ -37,12 +41,12 @@ public class StormUnionITCase extends StreamingProgramTestBase { + "-624140595\n" + "-60658084\n" + "142959438\n" + "-613647601\n" + "-330177159\n" + "-54027108\n" + "1945002173\n" + "979930868"; - private final static String topologyId = "Multiple Input Streams Test"; - private final static String spoutId1 = "spout1"; - private final static String spoutId2 = "spout2"; - private final static String spoutId3 = "spout3"; - private final static String boltId = "merger"; - private final static String sinkId = "sink"; + private static final String topologyId = "Multiple Input Streams Test"; + private static final String spoutId1 = "spout1"; + private static final String spoutId2 = "spout2"; + private static final String spoutId3 = "spout3"; + private static final String boltId = "merger"; + private static final String sinkId = "sink"; private String resultPath; @Override diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java index da8d21aa0274e6..923f9dff21043c 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/FiniteRandomSpout.java @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests.operators; -import java.util.Map; -import java.util.Random; +package org.apache.flink.storm.tests.operators; import org.apache.flink.storm.util.FiniteSpout; @@ -30,6 +28,13 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.Utils; +import java.util.Map; +import java.util.Random; + +/** + * A Spout implementation that broadcasts random numbers across a specified number of output streams, until a specified + * count is reached. + */ public class FiniteRandomSpout extends BaseRichSpout implements FiniteSpout { private static final long serialVersionUID = 6592885571932363239L; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java index 7a6ec584d107fa..c04bce308a1a59 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MergerBolt.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests.operators; -import java.util.Map; +package org.apache.flink.storm.tests.operators; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -26,6 +25,11 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; +import java.util.Map; + +/** + * A Bolt implementation that forwards all incoming tuples to a single output stream. + */ public class MergerBolt extends BaseRichBolt { private static final long serialVersionUID = -7966475984592762720L; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java index a6e19b9bb11e79..4061a711628881 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/MetaDataSpout.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests.operators; -import java.util.Map; +package org.apache.flink.storm.tests.operators; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; @@ -26,6 +25,11 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; +import java.util.Map; + +/** + * A Spout implementation emitting meta data. + */ public class MetaDataSpout extends BaseRichSpout { private static final long serialVersionUID = 5305870218033256376L; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java index fb5c8d30352264..1a8f218dbf0445 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/TaskIdBolt.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests.operators; -import java.util.Map; +package org.apache.flink.storm.tests.operators; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -27,6 +26,8 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; +import java.util.Map; + /** * Bolt to prepend all incoming tuple values with the task id. */ diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java index f7b40fcc429431..059882bdd09ce9 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/tests/operators/VerifyMetaDataBolt.java @@ -15,9 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.storm.tests.operators; -import java.util.Map; +package org.apache.flink.storm.tests.operators; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -28,6 +27,11 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; +import java.util.Map; + +/** + * A Bolt implementation that verifies metadata emitted by a {@link MetaDataSpout}. + */ public class VerifyMetaDataBolt extends BaseRichBolt { private static final long serialVersionUID = 1353222852073800478L; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java index f48e2f6832c4d7..d1cc5a282c59f9 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountITCase.java @@ -21,6 +21,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the BoltTokenizerWordCount example. + */ public class BoltTokenizerWordCountITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java index 902cacf15630e9..0eb4a6e95cb8d2 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountPojoITCase.java @@ -21,6 +21,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the BoltTokenizerWordCountPojo example. + */ public class BoltTokenizerWordCountPojoITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java index 160efb34bdd697..8879d9e50c3508 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/BoltTokenizerWordCountWithNamesITCase.java @@ -21,6 +21,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the BoltTokenizerWordCountWithNames example. + */ public class BoltTokenizerWordCountWithNamesITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java index 17f5be58cfd399..ec2ca2c62d8d56 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/SpoutSourceWordCountITCase.java @@ -21,6 +21,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the SpoutSourceWordCount example. + */ public class SpoutSourceWordCountITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java index 47ef2c4d125a0b..471afa93e51de5 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalITCase.java @@ -21,6 +21,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; +/** + * Test for the WordCountLocal example. + */ public class WordCountLocalITCase extends StreamingProgramTestBase { protected String textPath; diff --git a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java index b04faa5bbd912f..445ea37cbdaa85 100644 --- a/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java +++ b/flink-contrib/flink-storm-examples/src/test/java/org/apache/flink/storm/wordcount/WordCountLocalNamedITCase.java @@ -21,7 +21,9 @@ import org.apache.flink.streaming.util.StreamingProgramTestBase; import org.apache.flink.test.testdata.WordCountData; - +/** + * Test for the WordCountLocalByName example. + */ public class WordCountLocalNamedITCase extends StreamingProgramTestBase { protected String textPath; From 350d928ae67e3704f94d41e1fad5ebab2abbc8ef Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 24 May 2017 10:51:23 +0200 Subject: [PATCH 5/5] [FLINK-6695] Activate strict checkstyle for flink-streaming-contrib This closes #4004. --- flink-contrib/flink-streaming-contrib/pom.xml | 35 +++++++++++++++ .../flink/contrib/streaming/CollectSink.java | 18 ++++---- .../contrib/streaming/DataStreamUtils.java | 15 ++++--- .../streaming/SocketStreamIterator.java | 45 +++++++++---------- .../contrib/streaming/CollectITCase.java | 14 +++--- .../streaming/SocketStreamIteratorTest.java | 31 +++++++------ 6 files changed, 100 insertions(+), 58 deletions(-) diff --git a/flink-contrib/flink-streaming-contrib/pom.xml b/flink-contrib/flink-streaming-contrib/pom.xml index 2381d707ea6533..812043c42d5671 100644 --- a/flink-contrib/flink-streaming-contrib/pom.xml +++ b/flink-contrib/flink-streaming-contrib/pom.xml @@ -182,6 +182,41 @@ under the License. ${project.basedir}/../../tools/maven/scalastyle-config.xml + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + /tools/maven/strict-checkstyle.xml + /tools/maven/suppressions.xml + true + true + true + + + + + test-compile + + check + + + + diff --git a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/CollectSink.java b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/CollectSink.java index 161eb16ab9269c..13127fe095eec3 100644 --- a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/CollectSink.java +++ b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/CollectSink.java @@ -17,27 +17,27 @@ package org.apache.flink.contrib.streaming; -import java.io.IOException; -import java.io.OutputStream; -import java.net.Socket; -import java.net.InetAddress; - +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.api.common.typeutils.TypeSerializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.Socket; /** * A specialized data sink to be used by DataStreamUtils.collect. */ class CollectSink extends RichSinkFunction { - + private static final long serialVersionUID = 1L; private final InetAddress hostIp; private final int port; private final TypeSerializer serializer; - + private transient Socket client; private transient OutputStream outputStream; private transient DataOutputViewStreamWrapper streamWriter; @@ -91,7 +91,7 @@ public void close() throws Exception { outputStream.flush(); outputStream.close(); } - + // first regular attempt to cleanly close. Failing that will escalate if (client != null) { client.close(); diff --git a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/DataStreamUtils.java b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/DataStreamUtils.java index 2987597479c3d9..430c98cb893db7 100644 --- a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/DataStreamUtils.java +++ b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/DataStreamUtils.java @@ -31,6 +31,9 @@ import java.net.UnknownHostException; import java.util.Iterator; +/** + * A collection of utilities for {@link DataStream DataStreams}. + */ public final class DataStreamUtils { /** @@ -38,19 +41,19 @@ public final class DataStreamUtils { * @return The iterator */ public static Iterator collect(DataStream stream) throws IOException { - + TypeSerializer serializer = stream.getType().createSerializer( stream.getExecutionEnvironment().getConfig()); - + SocketStreamIterator iter = new SocketStreamIterator(serializer); //Find out what IP of us should be given to CollectSink, that it will be able to connect to StreamExecutionEnvironment env = stream.getExecutionEnvironment(); InetAddress clientAddress; - + if (env instanceof RemoteStreamEnvironment) { - String host = ((RemoteStreamEnvironment)env).getHost(); - int port = ((RemoteStreamEnvironment)env).getPort(); + String host = ((RemoteStreamEnvironment) env).getHost(); + int port = ((RemoteStreamEnvironment) env).getPort(); try { clientAddress = ConnectionUtils.findConnectingAddress(new InetSocketAddress(host, port), 2000, 400); } @@ -73,7 +76,7 @@ public static Iterator collect(DataStream stream) throws IOExcep sink.setParallelism(1); // It would not work if multiple instances would connect to the same port (new CallExecute(env, iter)).start(); - + return iter; } diff --git a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/SocketStreamIterator.java b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/SocketStreamIterator.java index c65be85ef2a4fb..fddfe4e841d575 100644 --- a/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/SocketStreamIterator.java +++ b/flink-contrib/flink-streaming-contrib/src/main/java/org/apache/flink/contrib/streaming/SocketStreamIterator.java @@ -17,47 +17,46 @@ package org.apache.flink.contrib.streaming; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; + +import java.io.EOFException; +import java.io.IOException; import java.net.InetAddress; +import java.net.ServerSocket; import java.net.Socket; import java.util.Iterator; -import java.net.ServerSocket; -import java.io.IOException; -import java.io.EOFException; import java.util.NoSuchElementException; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.core.memory.DataInputViewStreamWrapper; - /** * An iterator that returns the data from a socket stream. - * + * *

          The iterator's constructor opens a server socket. In the first call to {@link #next()} * or {@link #hasNext()}, the iterator waits for a socket to connect, and starts receiving, * deserializing, and returning the data from that socket. - * + * * @param The type of elements returned from the iterator. */ class SocketStreamIterator implements Iterator { - /** Server socket to listen at */ + /** Server socket to listen at. */ private final ServerSocket socket; - /** Serializer to deserialize stream */ + /** Serializer to deserialize stream. */ private final TypeSerializer serializer; - /** Set by the same thread that reads it */ + /** Set by the same thread that reads it. */ private DataInputViewStreamWrapper inStream; - /** Next element, handover from hasNext() to next() */ + /** Next element, handover from hasNext() to next(). */ private T next; - /** The socket for the specific stream */ + /** The socket for the specific stream. */ private Socket connectedSocket; - /** Async error, for example by the executor of the program that produces the stream */ + /** Async error, for example by the executor of the program that produces the stream. */ private volatile Throwable error; - SocketStreamIterator(TypeSerializer serializer) throws IOException { this.serializer = serializer; try { @@ -79,18 +78,18 @@ class SocketStreamIterator implements Iterator { public int getPort() { return socket.getLocalPort(); } - + public InetAddress getBindAddress() { return socket.getInetAddress(); } - + public void close() { if (connectedSocket != null) { try { connectedSocket.close(); } catch (Throwable ignored) {} } - + try { socket.close(); } catch (Throwable ignored) {} @@ -114,7 +113,7 @@ public boolean hasNext() { throw new RuntimeException("Failed to receive next element: " + e.getMessage(), e); } } - + return next != null; } @@ -145,18 +144,18 @@ private T readNextFromStream() throws Exception { connectedSocket = socket.accept(); inStream = new DataInputViewStreamWrapper(connectedSocket.getInputStream()); } - + return serializer.deserialize(inStream); } catch (EOFException e) { try { connectedSocket.close(); } catch (Throwable ignored) {} - + try { socket.close(); } catch (Throwable ignored) {} - + return null; } catch (Exception e) { @@ -173,7 +172,7 @@ private T readNextFromStream() throws Exception { // ------------------------------------------------------------------------ // errors // ------------------------------------------------------------------------ - + public void notifyOfError(Throwable error) { if (error != null && this.error == null) { this.error = error; diff --git a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java index f9b6a2130fa172..55a4df321b746d 100644 --- a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java +++ b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/CollectITCase.java @@ -29,7 +29,7 @@ import java.util.Iterator; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * This test verifies the behavior of DataStreamUtils.collect. @@ -45,18 +45,18 @@ public void testCollect() throws Exception { TestStreamEnvironment.setAsContext(cluster, 1); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - final long N = 10; - DataStream stream = env.generateSequence(1, N); - + + final long n = 10; + DataStream stream = env.generateSequence(1, n); + long i = 1; for (Iterator it = DataStreamUtils.collect(stream); it.hasNext(); ) { long x = it.next(); assertEquals("received wrong element", i, x); i++; } - - assertEquals("received wrong number of elements", N + 1, i); + + assertEquals("received wrong number of elements", n + 1, i); } finally { TestStreamEnvironment.unsetAsContext(); diff --git a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/SocketStreamIteratorTest.java b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/SocketStreamIteratorTest.java index f8739a368909c8..0693ce29bab8ba 100644 --- a/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/SocketStreamIteratorTest.java +++ b/flink-contrib/flink-streaming-contrib/src/test/java/org/apache/flink/contrib/streaming/SocketStreamIteratorTest.java @@ -28,31 +28,36 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicReference; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +/** + * Tests for the SocketStreamIterator. + */ public class SocketStreamIteratorTest { - + @Test public void testIterator() throws Exception { - + final AtomicReference error = new AtomicReference<>(); - + final long seed = new Random().nextLong(); final int numElements = 1000; - + final SocketStreamIterator iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE); - + Thread writer = new Thread() { - + @Override public void run() { try { try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort()); - DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) - { + DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) { + final TypeSerializer serializer = LongSerializer.INSTANCE; final Random rnd = new Random(seed); - + for (int i = 0; i < numElements; i++) { serializer.serialize(rnd.nextLong(), out); } @@ -63,16 +68,16 @@ public void run() { } } }; - + writer.start(); - + final Random validator = new Random(seed); for (int i = 0; i < numElements; i++) { assertTrue(iterator.hasNext()); assertTrue(iterator.hasNext()); assertEquals(validator.nextLong(), iterator.next().longValue()); } - + assertFalse(iterator.hasNext()); writer.join(); assertFalse(iterator.hasNext());