From ae03b873ac5a921058246f6c02c626c69a295b9a Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Mon, 18 May 2020 13:47:06 +0200 Subject: [PATCH 001/773] [hotfix][docs] Update docs/_config.yml for 1.11 release --- docs/_config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/_config.yml b/docs/_config.yml index d7df380c661da..4ea361187c192 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,12 +27,12 @@ # we change the version for the complete docs when forking of a release branch # etc. # The full version string as referenced in Maven (e.g. 1.2.1) -version: "1.11-SNAPSHOT" +version: "1.11.0" # For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot # release this should be the same as the regular version -version_title: "1.11-SNAPSHOT" +version_title: "1.11" # Branch on Github for this version -github_branch: "master" +github_branch: "release-1.11" # Plain Scala version is needed for e.g. the Gradle quickstart. scala_version: "2.11" @@ -51,14 +51,14 @@ github_url: "https://github.com/apache/flink" download_url: "https://flink.apache.org/downloads.html" # please use a protocol relative URL here -baseurl: //ci.apache.org/projects/flink/flink-docs-master +baseurl: //ci.apache.org/projects/flink/flink-docs-1.11 stable_baseurl: //ci.apache.org/projects/flink/flink-docs-stable -javadocs_baseurl: //ci.apache.org/projects/flink/flink-docs-master -pythondocs_baseurl: //ci.apache.org/projects/flink/flink-docs-master +javadocs_baseurl: //ci.apache.org/projects/flink/flink-docs-1.11 +pythondocs_baseurl: //ci.apache.org/projects/flink/flink-docs-1.11 # Flag whether this is a stable version or not. Used for the quickstart page. -is_stable: false +is_stable: true # Flag to indicate whether an outdated warning should be shown. show_outdated_warning: false From 53da48a1072f720af7d4b3eddf0d70d3cd117dff Mon Sep 17 00:00:00 2001 From: Yuan Mei Date: Mon, 18 May 2020 16:47:37 +0800 Subject: [PATCH 002/773] [FLINK-15670][core] Provide a utility function to flatten a recursive Properties to a first level property HashTable In some cases, KafkaProducer#propsToMap for example, Properties is used purely as a HashTable without considering its default properties. --- .../org/apache/flink/util/PropertiesUtil.java | 23 ++++++++ .../apache/flink/util/PropertiesUtilTest.java | 52 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 flink-core/src/test/java/org/apache/flink/util/PropertiesUtilTest.java diff --git a/flink-core/src/main/java/org/apache/flink/util/PropertiesUtil.java b/flink-core/src/main/java/org/apache/flink/util/PropertiesUtil.java index 37d8a421f96f0..f587c0a571d67 100644 --- a/flink-core/src/main/java/org/apache/flink/util/PropertiesUtil.java +++ b/flink-core/src/main/java/org/apache/flink/util/PropertiesUtil.java @@ -19,6 +19,7 @@ import org.slf4j.Logger; +import java.util.Collections; import java.util.Properties; /** @@ -108,6 +109,28 @@ public static boolean getBoolean(Properties config, String key, boolean defaultV } } + /** + * Flatten a recursive {@link Properties} to a first level property map. + * + *

In some cases, {@code KafkaProducer#propsToMap} for example, Properties is used purely as a HashTable + * without considering its default properties. + * + * @param config Properties to be flattened + * @return Properties without defaults; all properties are put in the first-level + */ + public static Properties flatten(Properties config) { + final Properties flattenProperties = new Properties(); + + Collections.list(config.propertyNames()).stream().forEach( + name -> { + Preconditions.checkArgument(name instanceof String); + flattenProperties.setProperty((String) name, config.getProperty((String) name)); + } + ); + + return flattenProperties; + } + // ------------------------------------------------------------------------ /** Private default constructor to prevent instantiation. */ diff --git a/flink-core/src/test/java/org/apache/flink/util/PropertiesUtilTest.java b/flink-core/src/test/java/org/apache/flink/util/PropertiesUtilTest.java new file mode 100644 index 0000000000000..80c2818c7d5e9 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/util/PropertiesUtilTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +import static org.apache.flink.util.PropertiesUtil.flatten; + +/** + * Tests for the {@link PropertiesUtil}. + */ +public class PropertiesUtilTest { + + @Test + public void testFlatten() { + // default Properties is null + Properties prop1 = new Properties(); + prop1.put("key1", "value1"); + + // default Properties is prop1 + Properties prop2 = new Properties(prop1); + prop2.put("key2", "value2"); + + // default Properties is prop2 + Properties prop3 = new Properties(prop2); + prop3.put("key3", "value3"); + + Properties flattened = flatten(prop3); + Assert.assertEquals(flattened.get("key1"), "value1"); + Assert.assertEquals(flattened.get("key2"), "value2"); + Assert.assertEquals(flattened.get("key3"), "value3"); + } +} From 2dca425e7fbcb4ed8f28f226e37a49cc58ea7035 Mon Sep 17 00:00:00 2001 From: Yuan Mei Date: Mon, 18 May 2020 16:54:26 +0800 Subject: [PATCH 003/773] [FLINK-15670][connector] Adds the producer for KafkaShuffle. KafkaShuffle provides a transparent Kafka source and sink pair, through which the network traffic of a shuffle step is persisted and redirected. --- .../connectors/kafka/FlinkKafkaProducer.java | 18 +- .../shuffle/FlinkKafkaShuffleProducer.java | 213 ++++++++++++++++++ 2 files changed, 224 insertions(+), 7 deletions(-) create mode 100644 flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleProducer.java diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java index d4b1b4f37b723..9d033b782c92b 100644 --- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer.java @@ -210,7 +210,7 @@ public enum Semantic { /** * The name of the default topic this producer is writing data to. */ - private final String defaultTopicId; + protected final String defaultTopicId; /** * (Serializable) SerializationSchema for turning objects used with Flink into. @@ -235,7 +235,7 @@ public enum Semantic { /** * Partitions of each topic. */ - private final Map topicPartitionsMap; + protected final Map topicPartitionsMap; /** * Max number of producers in the pool. If all producers are in use, snapshoting state will throw an exception. @@ -250,7 +250,7 @@ public enum Semantic { /** * Flag controlling whether we are writing the Flink record's timestamp into Kafka. */ - private boolean writeTimestampToKafka = false; + protected boolean writeTimestampToKafka = false; /** * Flag indicating whether to accept failures (and log them), or to fail on failures. @@ -273,7 +273,7 @@ public enum Semantic { protected transient volatile Exception asyncException; /** Number of unacknowledged records. */ - private final AtomicLong pendingRecords = new AtomicLong(); + protected final AtomicLong pendingRecords = new AtomicLong(); /** Cache of metrics to replace already registered metrics instead of overwriting existing ones. */ private final Map previouslyCreatedMetrics = new HashMap<>(); @@ -1214,7 +1214,7 @@ private FlinkKafkaInternalProducer initProducer(boolean register return producer; } - private void checkErroneous() throws FlinkKafkaException { + protected void checkErroneous() throws FlinkKafkaException { Exception e = asyncException; if (e != null) { // prevent double throwing @@ -1256,7 +1256,7 @@ private static Properties getPropertiesFromBrokerList(String brokerList) { return props; } - private static int[] getPartitionsByTopic(String topic, Producer producer) { + protected static int[] getPartitionsByTopic(String topic, Producer producer) { // the fetched list is immutable, so we're creating a mutable copy in order to sort it List partitionsList = new ArrayList<>(producer.partitionsFor(topic)); @@ -1281,7 +1281,7 @@ public int compare(PartitionInfo o1, PartitionInfo o2) { */ @VisibleForTesting @Internal - static class KafkaTransactionState { + protected static class KafkaTransactionState { private final transient FlinkKafkaInternalProducer producer; @@ -1315,6 +1315,10 @@ boolean isTransactional() { return transactionalId != null; } + public FlinkKafkaInternalProducer getProducer() { + return producer; + } + @Override public String toString() { return String.format( diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleProducer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleProducer.java new file mode 100644 index 0000000000000..6be2bba64a7d9 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleProducer.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.runtime.state.KeyGroupRangeAssignment; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaException; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.PropertiesUtil; + +import org.apache.kafka.clients.producer.ProducerRecord; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Properties; + +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffle.PARTITION_NUMBER; + +/** + * Flink Kafka Shuffle Producer Function. + * It is different from {@link FlinkKafkaProducer} in the way handling elements and watermarks + */ +@Internal +public class FlinkKafkaShuffleProducer extends FlinkKafkaProducer { + private final KafkaSerializer kafkaSerializer; + private final KeySelector keySelector; + private final int numberOfPartitions; + + FlinkKafkaShuffleProducer( + String defaultTopicId, + TypeSerializer typeSerializer, + Properties props, + KeySelector keySelector, + Semantic semantic, + int kafkaProducersPoolSize) { + super(defaultTopicId, (element, timestamp) -> null, props, semantic, kafkaProducersPoolSize); + + this.kafkaSerializer = new KafkaSerializer<>(typeSerializer); + this.keySelector = keySelector; + + Preconditions.checkArgument( + props.getProperty(PARTITION_NUMBER) != null, + "Missing partition number for Kafka Shuffle"); + numberOfPartitions = PropertiesUtil.getInt(props, PARTITION_NUMBER, Integer.MIN_VALUE); + } + + /** + * This is the function invoked to handle each element. + * + * @param transaction Transaction state; + * elements are written to Kafka in transactions to guarantee different level of data consistency + * @param next Element to handle + * @param context Context needed to handle the element + * @throws FlinkKafkaException for kafka error + */ + @Override + public void invoke(KafkaTransactionState transaction, IN next, Context context) throws FlinkKafkaException { + checkErroneous(); + + // write timestamp to Kafka if timestamp is available + Long timestamp = context.timestamp(); + + int[] partitions = getPartitions(transaction); + int partitionIndex; + try { + partitionIndex = KeyGroupRangeAssignment + .assignKeyToParallelOperator(keySelector.getKey(next), partitions.length, partitions.length); + } catch (Exception e) { + throw new RuntimeException("Fail to assign a partition number to record", e); + } + + ProducerRecord record = new ProducerRecord<>( + defaultTopicId, + partitionIndex, + timestamp, + null, + kafkaSerializer.serializeRecord(next, timestamp)); + + pendingRecords.incrementAndGet(); + transaction.getProducer().send(record, callback); + } + + /** + * This is the function invoked to handle each watermark. + * + * @param watermark Watermark to handle + * @throws FlinkKafkaException For kafka error + */ + public void invoke(Watermark watermark) throws FlinkKafkaException { + checkErroneous(); + KafkaTransactionState transaction = currentTransaction(); + + int[] partitions = getPartitions(transaction); + int subtask = getRuntimeContext().getIndexOfThisSubtask(); + + // broadcast watermark + long timestamp = watermark.getTimestamp(); + for (int partition : partitions) { + ProducerRecord record = new ProducerRecord<>( + defaultTopicId, + partition, + timestamp, + null, + kafkaSerializer.serializeWatermark(watermark, subtask)); + + pendingRecords.incrementAndGet(); + transaction.getProducer().send(record, callback); + } + } + + private int[] getPartitions(KafkaTransactionState transaction) { + int[] partitions = topicPartitionsMap.get(defaultTopicId); + if (partitions == null) { + partitions = getPartitionsByTopic(defaultTopicId, transaction.getProducer()); + topicPartitionsMap.put(defaultTopicId, partitions); + } + + Preconditions.checkArgument(partitions.length == numberOfPartitions); + + return partitions; + } + + /** + * Flink Kafka Shuffle Serializer. + */ + public static final class KafkaSerializer implements Serializable { + public static final int TAG_REC_WITH_TIMESTAMP = 0; + public static final int TAG_REC_WITHOUT_TIMESTAMP = 1; + public static final int TAG_WATERMARK = 2; + + private static final long serialVersionUID = 2000002L; + // easy for updating SerDe format later + private static final int KAFKA_SHUFFLE_VERSION = 0; + + private final TypeSerializer serializer; + + private transient DataOutputSerializer dos; + + KafkaSerializer(TypeSerializer serializer) { + this.serializer = serializer; + } + + /** + * Format: Version(byte), TAG(byte), [timestamp(long)], record. + */ + byte[] serializeRecord(IN record, Long timestamp) { + if (dos == null) { + dos = new DataOutputSerializer(16); + } + + try { + dos.write(KAFKA_SHUFFLE_VERSION); + + if (timestamp == null) { + dos.write(TAG_REC_WITHOUT_TIMESTAMP); + } else { + dos.write(TAG_REC_WITH_TIMESTAMP); + dos.writeLong(timestamp); + } + serializer.serialize(record, dos); + + } catch (IOException e) { + throw new RuntimeException("Unable to serialize record", e); + } + + byte[] ret = dos.getCopyOfBuffer(); + dos.clear(); + return ret; + } + + /** + * Format: Version(byte), TAG(byte), subtask(int), timestamp(long). + */ + byte[] serializeWatermark(Watermark watermark, int subtask) { + if (dos == null) { + dos = new DataOutputSerializer(16); + } + + try { + dos.write(KAFKA_SHUFFLE_VERSION); + dos.write(TAG_WATERMARK); + dos.writeInt(subtask); + dos.writeLong(watermark.getTimestamp()); + } catch (IOException e) { + throw new RuntimeException("Unable to serialize watermark", e); + } + + byte[] ret = dos.getCopyOfBuffer(); + dos.clear(); + return ret; + } + } +} From c3b42fb924d571c6a88223411b414969bf8b89d4 Mon Sep 17 00:00:00 2001 From: Yuan Mei Date: Mon, 18 May 2020 16:57:56 +0800 Subject: [PATCH 004/773] [FLINK-15670][connector] Adds the consumer for KafkaShuffle. KafkaShuffle provides a transparent Kafka source and sink pair, through which the network traffic of a shuffle step is persisted and redirected. --- .../kafka/FlinkKafkaConsumerBase.java | 2 +- .../kafka/internals/AbstractFetcher.java | 2 +- .../kafka/internal/KafkaFetcher.java | 55 ++-- .../kafka/internal/KafkaShuffleFetcher.java | 297 ++++++++++++++++++ .../shuffle/FlinkKafkaShuffleConsumer.java | 94 ++++++ 5 files changed, 424 insertions(+), 26 deletions(-) create mode 100644 flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaShuffleFetcher.java create mode 100644 flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleConsumer.java diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index 46688f68ae4b2..f9f835a0b7705 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -264,7 +264,7 @@ public FlinkKafkaConsumerBase( * @param properties - Kafka configuration properties to be adjusted * @param offsetCommitMode offset commit mode */ - static void adjustAutoCommitConfig(Properties properties, OffsetCommitMode offsetCommitMode) { + protected static void adjustAutoCommitConfig(Properties properties, OffsetCommitMode offsetCommitMode) { if (offsetCommitMode == OffsetCommitMode.ON_CHECKPOINTS || offsetCommitMode == OffsetCommitMode.DISABLED) { properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); } diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java index 9ad685cd42e8a..978cd97b07cfb 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java @@ -84,7 +84,7 @@ public abstract class AbstractFetcher { /** The lock that guarantees that record emission and state updates are atomic, * from the view of taking a checkpoint. */ - private final Object checkpointLock; + protected final Object checkpointLock; /** All partitions (and their state) that this fetcher is subscribed to. */ private final List> subscribedPartitionStates; diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaFetcher.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaFetcher.java index d591f584427c7..d2be85eeb3519 100644 --- a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaFetcher.java +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaFetcher.java @@ -63,14 +63,17 @@ public class KafkaFetcher extends AbstractFetcher { /** The schema to convert between Kafka's byte messages, and Flink's objects. */ private final KafkaDeserializationSchema deserializer; + /** A collector to emit records in batch (bundle). **/ + private final KafkaCollector kafkaCollector; + /** The handover of data and exceptions between the consumer thread and the task thread. */ - private final Handover handover; + final Handover handover; /** The thread that runs the actual KafkaConsumer and hand the record batches to this fetcher. */ - private final KafkaConsumerThread consumerThread; + final KafkaConsumerThread consumerThread; /** Flag to mark the main work loop as alive. */ - private volatile boolean running = true; + volatile boolean running = true; // ------------------------------------------------------------------------ @@ -111,19 +114,16 @@ public KafkaFetcher( useMetrics, consumerMetricGroup, subtaskMetricGroup); + this.kafkaCollector = new KafkaCollector(); } // ------------------------------------------------------------------------ // Fetcher work methods // ------------------------------------------------------------------------ - private final KafkaCollector kafkaCollector = new KafkaCollector(); - @Override public void runFetchLoop() throws Exception { try { - final Handover handover = this.handover; - // kick off the actual Kafka consumer consumerThread.start(); @@ -138,23 +138,7 @@ public void runFetchLoop() throws Exception { List> partitionRecords = records.records(partition.getKafkaPartitionHandle()); - for (ConsumerRecord record : partitionRecords) { - deserializer.deserialize(record, kafkaCollector); - - // emit the actual records. this also updates offset state atomically and emits - // watermarks - emitRecordsWithTimestamps( - kafkaCollector.getRecords(), - partition, - record.offset(), - record.timestamp()); - - if (kafkaCollector.isEndOfStreamSignalled()) { - // end of stream signaled - running = false; - break; - } - } + partitionConsumerRecordsHandler(partitionRecords, partition); } } } @@ -189,6 +173,29 @@ protected String getFetcherName() { return "Kafka Fetcher"; } + protected void partitionConsumerRecordsHandler( + List> partitionRecords, + KafkaTopicPartitionState partition) throws Exception { + + for (ConsumerRecord record : partitionRecords) { + deserializer.deserialize(record, kafkaCollector); + + // emit the actual records. this also updates offset state atomically and emits + // watermarks + emitRecordsWithTimestamps( + kafkaCollector.getRecords(), + partition, + record.offset(), + record.timestamp()); + + if (kafkaCollector.isEndOfStreamSignalled()) { + // end of stream signaled + running = false; + break; + } + } + } + // ------------------------------------------------------------------------ // Implement Methods of the AbstractFetcher // ------------------------------------------------------------------------ diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaShuffleFetcher.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaShuffleFetcher.java new file mode 100644 index 0000000000000..5d380dabd4545 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaShuffleFetcher.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.internal; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.ByteSerializer; +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionState; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.SerializedValue; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.TopicPartition; + +import java.io.Serializable; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffleProducer.KafkaSerializer.TAG_REC_WITHOUT_TIMESTAMP; +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffleProducer.KafkaSerializer.TAG_REC_WITH_TIMESTAMP; +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffleProducer.KafkaSerializer.TAG_WATERMARK; + +/** + * Fetch data from Kafka for Kafka Shuffle. + */ +@Internal +public class KafkaShuffleFetcher extends KafkaFetcher { + /** The handler to check and generate watermarks from fetched records. **/ + private final WatermarkHandler watermarkHandler; + + /** The schema to convert between Kafka's byte messages, and Flink's objects. */ + private final KafkaShuffleElementDeserializer kafkaShuffleDeserializer; + + public KafkaShuffleFetcher( + SourceFunction.SourceContext sourceContext, + Map assignedPartitionsWithInitialOffsets, + SerializedValue> watermarkStrategy, + ProcessingTimeService processingTimeProvider, + long autoWatermarkInterval, + ClassLoader userCodeClassLoader, + String taskNameWithSubtasks, + KafkaDeserializationSchema deserializer, + Properties kafkaProperties, + long pollTimeout, + MetricGroup subtaskMetricGroup, + MetricGroup consumerMetricGroup, + boolean useMetrics, + TypeSerializer typeSerializer, + int producerParallelism) throws Exception { + super( + sourceContext, + assignedPartitionsWithInitialOffsets, + watermarkStrategy, + processingTimeProvider, + autoWatermarkInterval, + userCodeClassLoader, + taskNameWithSubtasks, + deserializer, + kafkaProperties, + pollTimeout, + subtaskMetricGroup, + consumerMetricGroup, + useMetrics); + + this.kafkaShuffleDeserializer = new KafkaShuffleElementDeserializer<>(typeSerializer); + this.watermarkHandler = new WatermarkHandler(producerParallelism); + } + + @Override + protected String getFetcherName() { + return "Kafka Shuffle Fetcher"; + } + + @Override + protected void partitionConsumerRecordsHandler( + List> partitionRecords, + KafkaTopicPartitionState partition) throws Exception { + + for (ConsumerRecord record : partitionRecords) { + final KafkaShuffleElement element = kafkaShuffleDeserializer.deserialize(record); + + // TODO: Do we need to check the end of stream if reaching the end watermark + // TODO: Currently, if one of the partition sends an end-of-stream signal the fetcher stops running. + // The current "ending of stream" logic in KafkaFetcher a bit strange: if any partition has a record + // signaled as "END_OF_STREAM", the fetcher will stop running. Notice that the signal is coming from + // the deserializer, which means from Kafka data itself. But it is possible that other topics + // and partitions still have data to read. Finishing reading Partition0 can not guarantee that Partition1 + // also finishes. + if (element.isRecord()) { + // timestamp is inherent from upstream + // If using ProcessTime, timestamp is going to be ignored (upstream does not include timestamp as well) + // If using IngestionTime, timestamp is going to be overwritten + // If using EventTime, timestamp is going to be used + synchronized (checkpointLock) { + KafkaShuffleRecord elementAsRecord = element.asRecord(); + sourceContext.collectWithTimestamp( + elementAsRecord.value, + elementAsRecord.timestamp == null ? record.timestamp() : elementAsRecord.timestamp); + partition.setOffset(record.offset()); + } + } else if (element.isWatermark()) { + final KafkaShuffleWatermark watermark = element.asWatermark(); + Optional newWatermark = watermarkHandler.checkAndGetNewWatermark(watermark); + newWatermark.ifPresent(sourceContext::emitWatermark); + } + } + } + + /** + * An element in a KafkaShuffle. Can be a record or a Watermark. + */ + @VisibleForTesting + public abstract static class KafkaShuffleElement { + + public boolean isRecord() { + return getClass() == KafkaShuffleRecord.class; + } + + public boolean isWatermark() { + return getClass() == KafkaShuffleWatermark.class; + } + + public KafkaShuffleRecord asRecord() { + return (KafkaShuffleRecord) this; + } + + public KafkaShuffleWatermark asWatermark() { + return (KafkaShuffleWatermark) this; + } + } + + /** + * A watermark element in a KafkaShuffle. It includes + * - subtask index where the watermark is coming from + * - watermark timestamp + */ + @VisibleForTesting + public static class KafkaShuffleWatermark extends KafkaShuffleElement { + final int subtask; + final long watermark; + + KafkaShuffleWatermark(int subtask, long watermark) { + this.subtask = subtask; + this.watermark = watermark; + } + + public int getSubtask() { + return subtask; + } + + public long getWatermark() { + return watermark; + } + } + + /** + * One value with Type T in a KafkaShuffle. This stores the value and an optional associated timestamp. + */ + @VisibleForTesting + public static class KafkaShuffleRecord extends KafkaShuffleElement { + final T value; + final Long timestamp; + + KafkaShuffleRecord(T value) { + this.value = value; + this.timestamp = null; + } + + KafkaShuffleRecord(long timestamp, T value) { + this.value = value; + this.timestamp = timestamp; + } + + public T getValue() { + return value; + } + + public Long getTimestamp() { + return timestamp; + } + } + + /** + * Deserializer for KafkaShuffleElement. + */ + @VisibleForTesting + public static class KafkaShuffleElementDeserializer implements Serializable { + private static final long serialVersionUID = 1000001L; + + private final TypeSerializer typeSerializer; + + private transient DataInputDeserializer dis; + + @VisibleForTesting + public KafkaShuffleElementDeserializer(TypeSerializer typeSerializer) { + this.typeSerializer = typeSerializer; + } + + @VisibleForTesting + public KafkaShuffleElement deserialize(ConsumerRecord record) + throws Exception { + byte[] value = record.value(); + + if (dis != null) { + dis.setBuffer(value); + } else { + dis = new DataInputDeserializer(value); + } + + // version byte + ByteSerializer.INSTANCE.deserialize(dis); + int tag = ByteSerializer.INSTANCE.deserialize(dis); + + if (tag == TAG_REC_WITHOUT_TIMESTAMP) { + return new KafkaShuffleRecord<>(typeSerializer.deserialize(dis)); + } else if (tag == TAG_REC_WITH_TIMESTAMP) { + return new KafkaShuffleRecord<>( + LongSerializer.INSTANCE.deserialize(dis), + typeSerializer.deserialize(dis)); + } else if (tag == TAG_WATERMARK) { + return new KafkaShuffleWatermark( + IntSerializer.INSTANCE.deserialize(dis), LongSerializer.INSTANCE.deserialize(dis)); + } + + throw new UnsupportedOperationException("Unsupported tag format"); + } + } + + /** + * WatermarkHandler to check and generate watermarks from fetched records. + */ + private static class WatermarkHandler { + private final int producerParallelism; + private final Map subtaskWatermark; + + private long currentMinWatermark = Long.MIN_VALUE; + + WatermarkHandler(int producerParallelism) { + this.producerParallelism = producerParallelism; + this.subtaskWatermark = new HashMap<>(producerParallelism); + } + + private Optional checkAndGetNewWatermark(KafkaShuffleWatermark newWatermark) { + // watermarks is incremental for the same partition and PRODUCER subtask + Long currentSubTaskWatermark = subtaskWatermark.get(newWatermark.subtask); + + // watermark is strictly increasing + Preconditions.checkState( + (currentSubTaskWatermark == null) || (currentSubTaskWatermark < newWatermark.watermark), + "Watermark should always increase: current : new " + currentSubTaskWatermark + ":" + newWatermark.watermark); + + subtaskWatermark.put(newWatermark.subtask, newWatermark.watermark); + + if (subtaskWatermark.values().size() < producerParallelism) { + return Optional.empty(); + } + + long minWatermark = subtaskWatermark.values().stream().min(Comparator.naturalOrder()).orElse(Long.MIN_VALUE); + if (currentMinWatermark < minWatermark) { + currentMinWatermark = minWatermark; + return Optional.of(new Watermark(minWatermark)); + } else { + return Optional.empty(); + } + } + } +} diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleConsumer.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleConsumer.java new file mode 100644 index 0000000000000..6403f427bbc5b --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffleConsumer.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; +import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode; +import org.apache.flink.streaming.connectors.kafka.internal.KafkaShuffleFetcher; +import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.PropertiesUtil; +import org.apache.flink.util.SerializedValue; + +import java.util.Map; +import java.util.Properties; + +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffle.PRODUCER_PARALLELISM; + +/** + * Flink Kafka Shuffle Consumer Function. + */ +@Internal +public class FlinkKafkaShuffleConsumer extends FlinkKafkaConsumer { + private final TypeSerializer typeSerializer; + private final int producerParallelism; + + FlinkKafkaShuffleConsumer( + String topic, + TypeInformationSerializationSchema schema, + TypeSerializer typeSerializer, + Properties props) { + // The schema is needed to call the right FlinkKafkaConsumer constructor. + // It is never used, can be `null`, but `null` confuses the compiler. + super(topic, schema, props); + this.typeSerializer = typeSerializer; + + Preconditions.checkArgument( + props.getProperty(PRODUCER_PARALLELISM) != null, + "Missing producer parallelism for Kafka Shuffle"); + producerParallelism = PropertiesUtil.getInt(props, PRODUCER_PARALLELISM, Integer.MAX_VALUE); + } + + @Override + protected AbstractFetcher createFetcher( + SourceContext sourceContext, + Map assignedPartitionsWithInitialOffsets, + SerializedValue> watermarkStrategy, + StreamingRuntimeContext runtimeContext, + OffsetCommitMode offsetCommitMode, + MetricGroup consumerMetricGroup, + boolean useMetrics) throws Exception { + // make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS; + // this overwrites whatever setting the user configured in the properties + adjustAutoCommitConfig(properties, offsetCommitMode); + + return new KafkaShuffleFetcher<>( + sourceContext, + assignedPartitionsWithInitialOffsets, + watermarkStrategy, + runtimeContext.getProcessingTimeService(), + runtimeContext.getExecutionConfig().getAutoWatermarkInterval(), + runtimeContext.getUserCodeClassLoader(), + runtimeContext.getTaskNameWithSubtasks(), + deserializer, + properties, + pollTimeout, + runtimeContext.getMetricGroup(), + consumerMetricGroup, + useMetrics, + typeSerializer, + producerParallelism); + } +} From e2305f234d6f88c503aaef1576d6b06910bbf1ea Mon Sep 17 00:00:00 2001 From: Yuan Mei Date: Mon, 18 May 2020 17:05:12 +0800 Subject: [PATCH 005/773] [FLINK-15670][connector] Kafka Shuffle API Part KafkaShuffle provides a transparent Kafka source and sink pair, through which the network traffic of a shuffle step is persisted and redirected. --- .../kafka/shuffle/FlinkKafkaShuffle.java | 391 ++++++++++++++++++ .../kafka/shuffle/StreamKafkaShuffleSink.java | 43 ++ 2 files changed, 434 insertions(+) create mode 100644 flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffle.java create mode 100644 flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/StreamKafkaShuffleSink.java diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffle.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffle.java new file mode 100644 index 0000000000000..6408360f33887 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/FlinkKafkaShuffle.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.operators.Keys; +import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; +import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.runtime.state.KeyGroupRangeAssignment; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamUtils; +import org.apache.flink.streaming.api.datastream.KeyedStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.api.transformations.SinkTransformation; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; +import org.apache.flink.streaming.util.keys.KeySelectorUtil; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.PropertiesUtil; + +import java.util.Properties; + +/** + * {@link FlinkKafkaShuffle} uses Kafka as a message bus to shuffle and persist data at the same time. + * + *

Persisting shuffle data is useful when + * - you would like to reuse the shuffle data and/or, + * - you would like to avoid a full restart of a pipeline during failure recovery + * + *

Persisting shuffle is achieved by wrapping a {@link FlinkKafkaShuffleProducer} and + * a {@link FlinkKafkaShuffleConsumer} together into a {@link FlinkKafkaShuffle}. + * Here is an example how to use a {@link FlinkKafkaShuffle}. + * + * 

{@code
+ *	StreamExecutionEnvironment env = ... 					// create execution environment
+ * 	DataStream source = env.addSource(...)				// add data stream source
+ * 	DataStream dataStream = ...							// some transformation(s) based on source
+ *
+ *	KeyedStream keyedStream = FlinkKafkaShuffle
+ *		.persistentKeyBy(									// keyBy shuffle through kafka
+ * 			dataStream,										// data stream to be shuffled
+ * 			topic,											// Kafka topic written to
+ * 			producerParallelism,							// the number of tasks of a Kafka Producer
+ * 			numberOfPartitions,								// the number of partitions of the Kafka topic written to
+ * 			kafkaProperties,								// kafka properties for Kafka Producer and Consumer
+ * 			keySelector);							// key selector to retrieve key from `dataStream'
+ *
+ *	keyedStream.transform...								// some other transformation(s)
+ *
+ * 	KeyedStream keyedStreamReuse = FlinkKafkaShuffle
+ * 		.readKeyBy(											// Read the Kafka shuffle data again for other usages
+ * 			topic,											// the topic of Kafka where data is persisted
+ * 			env,											// execution environment, and it can be a new environment
+ * 			typeInformation,								// type information of the data persisted in Kafka
+ * 			kafkaProperties,								// kafka properties for Kafka Consumer
+ * 			keySelector);							// key selector to retrieve key
+ *
+ * 	keyedStreamReuse.transform...							// some other transformation(s)
+ * }
+ * + *

Usage of {@link FlinkKafkaShuffle#persistentKeyBy} is similar to {@link DataStream#keyBy(KeySelector)}. + * The differences are: + * + *

1). Partitioning is done through {@link FlinkKafkaShuffleProducer}. {@link FlinkKafkaShuffleProducer} decides + * which partition a key goes when writing to Kafka + * + *

2). Shuffle data can be reused through {@link FlinkKafkaShuffle#readKeyBy}, as shown in the example above. + * + *

3). Job execution is decoupled by the persistent Kafka message bus. In the example, the job execution graph is + * decoupled to three regions: `KafkaShuffleProducer', `KafkaShuffleConsumer' and `KafkaShuffleConsumerReuse' + * through `PERSISTENT DATA` as shown below. If any region fails the execution, the other two keep progressing. + * + * 

+ *     source -> ... KafkaShuffleProducer -> PERSISTENT DATA -> KafkaShuffleConsumer -> ...
+ *                                                |
+ *                                                | ----------> KafkaShuffleConsumerReuse -> ...
+ *
+ */ +@Experimental +public class FlinkKafkaShuffle { + static final String PRODUCER_PARALLELISM = "producer parallelism"; + static final String PARTITION_NUMBER = "partition number"; + + /** + * Uses Kafka as a message bus to persist keyBy shuffle. + * + *

Persisting keyBy shuffle is achieved by wrapping a {@link FlinkKafkaShuffleProducer} and + * {@link FlinkKafkaShuffleConsumer} together. + * + *

On the producer side, {@link FlinkKafkaShuffleProducer} + * is similar to {@link DataStream#keyBy(KeySelector)}. They use the same key group assignment function + * {@link KeyGroupRangeAssignment#assignKeyToParallelOperator} to decide which partition a key goes. + * Hence, each producer task can potentially write to each Kafka partition based on where the key goes. + * Here, `numberOfPartitions` equals to the key group size. + * In the case of using {@link TimeCharacteristic#EventTime}, each producer task broadcasts its watermark + * to ALL of the Kafka partitions to make sure watermark information is propagated correctly. + * + *

On the consumer side, each consumer task should read partitions equal to the key group indices + * it is assigned. `numberOfPartitions` is the maximum parallelism of the consumer. This version only + * supports numberOfPartitions = consumerParallelism. + * In the case of using {@link TimeCharacteristic#EventTime}, a consumer task is responsible to emit + * watermarks. Watermarks are read from the corresponding Kafka partitions. Notice that a consumer task only starts + * to emit a watermark after reading at least one watermark from each producer task to make sure watermarks + * are monotonically increasing. Hence a consumer task needs to know `producerParallelism` as well. + * + * @see FlinkKafkaShuffle#writeKeyBy + * @see FlinkKafkaShuffle#readKeyBy + * + * @param dataStream Data stream to be shuffled + * @param topic Kafka topic written to + * @param producerParallelism Parallelism of producer + * @param numberOfPartitions Number of partitions + * @param properties Kafka properties + * @param keySelector Key selector to retrieve key from `dataStream' + * @param Type of the input data stream + * @param Type of key + */ + public static KeyedStream persistentKeyBy( + DataStream dataStream, + String topic, + int producerParallelism, + int numberOfPartitions, + Properties properties, + KeySelector keySelector) { + // KafkaProducer#propsToMap uses Properties purely as a HashMap without considering the default properties + // So we have to flatten the default property to first level elements. + Properties kafkaProperties = PropertiesUtil.flatten(properties); + kafkaProperties.setProperty(PRODUCER_PARALLELISM, String.valueOf(producerParallelism)); + kafkaProperties.setProperty(PARTITION_NUMBER, String.valueOf(numberOfPartitions)); + + StreamExecutionEnvironment env = dataStream.getExecutionEnvironment(); + + writeKeyBy(dataStream, topic, kafkaProperties, keySelector); + return readKeyBy(topic, env, dataStream.getType(), kafkaProperties, keySelector); + } + + /** + * Uses Kafka as a message bus to persist keyBy shuffle. + * + *

Persisting keyBy shuffle is achieved by wrapping a {@link FlinkKafkaShuffleProducer} and + * {@link FlinkKafkaShuffleConsumer} together. + * + *

On the producer side, {@link FlinkKafkaShuffleProducer} + * is similar to {@link DataStream#keyBy(KeySelector)}. They use the same key group assignment function + * {@link KeyGroupRangeAssignment#assignKeyToParallelOperator} to decide which partition a key goes. + * Hence, each producer task can potentially write to each Kafka partition based on where the key goes. + * Here, `numberOfPartitions` equals to the key group size. + * In the case of using {@link TimeCharacteristic#EventTime}, each producer task broadcasts its watermark + * to ALL of the Kafka partitions to make sure watermark information is propagated correctly. + * + *

On the consumer side, each consumer task should read partitions equal to the key group indices + * it is assigned. `numberOfPartitions` is the maximum parallelism of the consumer. This version only + * supports numberOfPartitions = consumerParallelism. + * In the case of using {@link TimeCharacteristic#EventTime}, a consumer task is responsible to emit + * watermarks. Watermarks are read from the corresponding Kafka partitions. Notice that a consumer task only starts + * to emit a watermark after reading at least one watermark from each producer task to make sure watermarks + * are monotonically increasing. Hence a consumer task needs to know `producerParallelism` as well. + * + * @see FlinkKafkaShuffle#writeKeyBy + * @see FlinkKafkaShuffle#readKeyBy + * + * @param dataStream Data stream to be shuffled + * @param topic Kafka topic written to + * @param producerParallelism Parallelism of producer + * @param numberOfPartitions Number of partitions + * @param properties Kafka properties + * @param fields Key positions from the input data stream + * @param Type of the input data stream + */ + public static KeyedStream persistentKeyBy( + DataStream dataStream, + String topic, + int producerParallelism, + int numberOfPartitions, + Properties properties, + int... fields) { + return persistentKeyBy( + dataStream, + topic, + producerParallelism, + numberOfPartitions, + properties, + keySelector(dataStream, fields)); + } + + /** + * The write side of {@link FlinkKafkaShuffle#persistentKeyBy}. + * + *

This function contains a {@link FlinkKafkaShuffleProducer} to shuffle and persist data in Kafka. + * {@link FlinkKafkaShuffleProducer} uses the same key group assignment function + * {@link KeyGroupRangeAssignment#assignKeyToParallelOperator} to decide which partition a key goes. + * Hence, each producer task can potentially write to each Kafka partition based on the key. + * Here, the number of partitions equals to the key group size. + * In the case of using {@link TimeCharacteristic#EventTime}, each producer task broadcasts each watermark + * to all of the Kafka partitions to make sure watermark information is propagated properly. + * + *

Attention: make sure kafkaProperties include + * {@link FlinkKafkaShuffle#PRODUCER_PARALLELISM} and {@link FlinkKafkaShuffle#PARTITION_NUMBER} explicitly. + * {@link FlinkKafkaShuffle#PRODUCER_PARALLELISM} is the parallelism of the producer. + * {@link FlinkKafkaShuffle#PARTITION_NUMBER} is the number of partitions. + * They are not necessarily the same and allowed to be set independently. + * + * @see FlinkKafkaShuffle#persistentKeyBy + * @see FlinkKafkaShuffle#readKeyBy + * + * @param dataStream Data stream to be shuffled + * @param topic Kafka topic written to + * @param kafkaProperties Kafka properties for Kafka Producer + * @param keySelector Key selector to retrieve key from `dataStream' + * @param Type of the input data stream + * @param Type of key + */ + public static void writeKeyBy( + DataStream dataStream, + String topic, + Properties kafkaProperties, + KeySelector keySelector) { + + StreamExecutionEnvironment env = dataStream.getExecutionEnvironment(); + TypeSerializer typeSerializer = dataStream.getType().createSerializer(env.getConfig()); + + // write data to Kafka + FlinkKafkaShuffleProducer kafkaProducer = new FlinkKafkaShuffleProducer<>( + topic, + typeSerializer, + kafkaProperties, + env.clean(keySelector), + FlinkKafkaProducer.Semantic.EXACTLY_ONCE, + FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE); + + // make sure the sink parallelism is set to producerParallelism + Preconditions.checkArgument( + kafkaProperties.getProperty(PRODUCER_PARALLELISM) != null, + "Missing producer parallelism for Kafka Shuffle"); + int producerParallelism = PropertiesUtil.getInt(kafkaProperties, PRODUCER_PARALLELISM, Integer.MIN_VALUE); + + addKafkaShuffle(dataStream, kafkaProducer, producerParallelism); + } + + /** + * The write side of {@link FlinkKafkaShuffle#persistentKeyBy}. + * + *

This function contains a {@link FlinkKafkaShuffleProducer} to shuffle and persist data in Kafka. + * {@link FlinkKafkaShuffleProducer} uses the same key group assignment function + * {@link KeyGroupRangeAssignment#assignKeyToParallelOperator} to decide which partition a key goes. + * + *

Hence, each producer task can potentially write to each Kafka partition based on the key. + * Here, the number of partitions equals to the key group size. + * In the case of using {@link TimeCharacteristic#EventTime}, each producer task broadcasts each watermark + * to all of the Kafka partitions to make sure watermark information is propagated properly. + * + *

Attention: make sure kafkaProperties include + * {@link FlinkKafkaShuffle#PRODUCER_PARALLELISM} and {@link FlinkKafkaShuffle#PARTITION_NUMBER} explicitly. + * {@link FlinkKafkaShuffle#PRODUCER_PARALLELISM} is the parallelism of the producer. + * {@link FlinkKafkaShuffle#PARTITION_NUMBER} is the number of partitions. + * They are not necessarily the same and allowed to be set independently. + * + * @see FlinkKafkaShuffle#persistentKeyBy + * @see FlinkKafkaShuffle#readKeyBy + * + * @param dataStream Data stream to be shuffled + * @param topic Kafka topic written to + * @param kafkaProperties Kafka properties for Kafka Producer + * @param fields Key positions from the input data stream + * @param Type of the input data stream + */ + public static void writeKeyBy( + DataStream dataStream, + String topic, + Properties kafkaProperties, + int... fields) { + writeKeyBy(dataStream, topic, kafkaProperties, keySelector(dataStream, fields)); + } + + /** + * The read side of {@link FlinkKafkaShuffle#persistentKeyBy}. + * + *

Each consumer task should read kafka partitions equal to the key group indices it is assigned. + * The number of kafka partitions is the maximum parallelism of the consumer. + * This version only supports numberOfPartitions = consumerParallelism. + * In the case of using {@link TimeCharacteristic#EventTime}, a consumer task is responsible to emit + * watermarks. Watermarks are read from the corresponding Kafka partitions. Notice that a consumer task only starts + * to emit a watermark after receiving at least one watermark from each producer task to make sure watermarks + * are monotonically increasing. Hence a consumer task needs to know `producerParallelism` as well. + * + *

Attention: make sure kafkaProperties include + * {@link FlinkKafkaShuffle#PRODUCER_PARALLELISM} and {@link FlinkKafkaShuffle#PARTITION_NUMBER} explicitly. + * {@link FlinkKafkaShuffle#PRODUCER_PARALLELISM} is the parallelism of the producer. + * {@link FlinkKafkaShuffle#PARTITION_NUMBER} is the number of partitions. + * They are not necessarily the same and allowed to be set independently. + * + * @see FlinkKafkaShuffle#persistentKeyBy + * @see FlinkKafkaShuffle#writeKeyBy + * + * @param topic The topic of Kafka where data is persisted + * @param env Execution environment. readKeyBy's environment can be different from writeKeyBy's + * @param typeInformation Type information of the data persisted in Kafka + * @param kafkaProperties kafka properties for Kafka Consumer + * @param keySelector key selector to retrieve key + * @param Schema type + * @param Key type + * @return Keyed data stream + */ + public static KeyedStream readKeyBy( + String topic, + StreamExecutionEnvironment env, + TypeInformation typeInformation, + Properties kafkaProperties, + KeySelector keySelector) { + + TypeSerializer typeSerializer = typeInformation.createSerializer(env.getConfig()); + TypeInformationSerializationSchema schema = + new TypeInformationSerializationSchema<>(typeInformation, typeSerializer); + + SourceFunction kafkaConsumer = + new FlinkKafkaShuffleConsumer<>(topic, schema, typeSerializer, kafkaProperties); + + // TODO: consider situations where numberOfPartitions != consumerParallelism + Preconditions.checkArgument( + kafkaProperties.getProperty(PARTITION_NUMBER) != null, + "Missing partition number for Kafka Shuffle"); + int numberOfPartitions = PropertiesUtil.getInt(kafkaProperties, PARTITION_NUMBER, Integer.MIN_VALUE); + DataStream outputDataStream = env.addSource(kafkaConsumer).setParallelism(numberOfPartitions); + + return DataStreamUtils.reinterpretAsKeyedStream(outputDataStream, keySelector); + } + + /** + * Adds a {@link StreamKafkaShuffleSink} to {@link DataStream}. + * + *

{@link StreamKafkaShuffleSink} is associated a {@link FlinkKafkaShuffleProducer}. + * + * @param inputStream Input data stream connected to the shuffle + * @param kafkaShuffleProducer Kafka shuffle sink function that can handle both records and watermark + * @param producerParallelism The number of tasks writing to the kafka shuffle + */ + private static void addKafkaShuffle( + DataStream inputStream, + FlinkKafkaShuffleProducer kafkaShuffleProducer, + int producerParallelism) { + + // read the output type of the input Transform to coax out errors about MissingTypeInfo + inputStream.getTransformation().getOutputType(); + + StreamKafkaShuffleSink shuffleSinkOperator = new StreamKafkaShuffleSink<>(kafkaShuffleProducer); + SinkTransformation transformation = new SinkTransformation<>( + inputStream.getTransformation(), + "kafka_shuffle", + shuffleSinkOperator, + inputStream.getExecutionEnvironment().getParallelism()); + inputStream.getExecutionEnvironment().addOperator(transformation); + transformation.setParallelism(producerParallelism); + } + + // A better place to put this function is DataStream; but put it here for now to avoid changing DataStream + private static KeySelector keySelector(DataStream source, int... fields) { + KeySelector keySelector; + if (source.getType() instanceof BasicArrayTypeInfo || source.getType() instanceof PrimitiveArrayTypeInfo) { + keySelector = KeySelectorUtil.getSelectorForArray(fields, source.getType()); + } else { + Keys keys = new Keys.ExpressionKeys<>(fields, source.getType()); + keySelector = KeySelectorUtil.getSelectorForKeys( + keys, + source.getType(), + source.getExecutionEnvironment().getConfig()); + } + + return keySelector; + } +} diff --git a/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/StreamKafkaShuffleSink.java b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/StreamKafkaShuffleSink.java new file mode 100644 index 0000000000000..ddb3e07b0040b --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/shuffle/StreamKafkaShuffleSink.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamSink; +import org.apache.flink.streaming.api.watermark.Watermark; + +/** + * A customized {@link StreamOperator} for executing {@link FlinkKafkaShuffleProducer} that handle + * both elements and watermarks. If the shuffle sink is determined to be useful to other sinks in the future, + * we should abstract this operator to data stream api. For now, we keep the operator this way to avoid + * public interface change. + */ +@Internal +class StreamKafkaShuffleSink extends StreamSink { + + public StreamKafkaShuffleSink(FlinkKafkaShuffleProducer flinkKafkaShuffleProducer) { + super(flinkKafkaShuffleProducer); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + super.processWatermark(mark); + ((FlinkKafkaShuffleProducer) userFunction).invoke(mark); + } +} From 37f6db04de7c0bc092fa07e821f7082549b167ef Mon Sep 17 00:00:00 2001 From: Yuan Mei Date: Mon, 18 May 2020 17:06:03 +0800 Subject: [PATCH 006/773] [FLINK-15670] Kafka Shuffle Test Case + add log4j2 file KafkaShuffle provides a transparent Kafka source and sink pair, through which the network traffic of a shuffle step is persisted and redirected. --- .../KafkaShuffleExactlyOnceITCase.java | 205 ++++++++ .../kafka/shuffle/KafkaShuffleITCase.java | 476 ++++++++++++++++++ .../kafka/shuffle/KafkaShuffleTestBase.java | 269 ++++++++++ .../src/test/resources/log4j2-test.properties | 38 ++ 4 files changed, 988 insertions(+) create mode 100644 flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleExactlyOnceITCase.java create mode 100644 flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleITCase.java create mode 100644 flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleTestBase.java create mode 100644 flink-connectors/flink-connector-kafka/src/test/resources/log4j2-test.properties diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleExactlyOnceITCase.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleExactlyOnceITCase.java new file mode 100644 index 0000000000000..84068620d22a6 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleExactlyOnceITCase.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.datastream.KeyedStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.kafka.testutils.FailingIdentityMapper; +import org.apache.flink.streaming.connectors.kafka.testutils.ValidatingExactlyOnceSink; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import static org.apache.flink.streaming.api.TimeCharacteristic.EventTime; +import static org.apache.flink.streaming.api.TimeCharacteristic.IngestionTime; +import static org.apache.flink.streaming.api.TimeCharacteristic.ProcessingTime; +import static org.apache.flink.test.util.TestUtils.tryExecute; + +/** + * Failure Recovery IT Test for KafkaShuffle. + */ +public class KafkaShuffleExactlyOnceITCase extends KafkaShuffleTestBase { + + @Rule + public final Timeout timeout = Timeout.millis(600000L); + + /** + * Failure Recovery after processing 2/3 data with time characteristic: ProcessingTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testFailureRecoveryProcessingTime() throws Exception { + testKafkaShuffleFailureRecovery(1000, ProcessingTime); + } + + /** + * Failure Recovery after processing 2/3 data with time characteristic: IngestionTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testFailureRecoveryIngestionTime() throws Exception { + testKafkaShuffleFailureRecovery(1000, IngestionTime); + } + + /** + * Failure Recovery after processing 2/3 data with time characteristic: EventTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testFailureRecoveryEventTime() throws Exception { + testKafkaShuffleFailureRecovery(1000, EventTime); + } + + /** + * Failure Recovery after data is repartitioned with time characteristic: ProcessingTime. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testAssignedToPartitionFailureRecoveryProcessingTime() throws Exception { + testAssignedToPartitionFailureRecovery(500, ProcessingTime); + } + + /** + * Failure Recovery after data is repartitioned with time characteristic: IngestionTime. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testAssignedToPartitionFailureRecoveryIngestionTime() throws Exception { + testAssignedToPartitionFailureRecovery(500, IngestionTime); + } + + /** + * Failure Recovery after data is repartitioned with time characteristic: EventTime. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testAssignedToPartitionFailureRecoveryEventTime() throws Exception { + testAssignedToPartitionFailureRecovery(500, EventTime); + } + + /** + * To test failure recovery after processing 2/3 data. + * + *

Schema: (key, timestamp, source instance Id). + * Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1 + */ + private void testKafkaShuffleFailureRecovery( + int numElementsPerProducer, + TimeCharacteristic timeCharacteristic) throws Exception { + + String topic = topic("failure_recovery", timeCharacteristic); + final int numberOfPartitions = 1; + final int producerParallelism = 1; + final int failAfterElements = numElementsPerProducer * numberOfPartitions * 2 / 3; + + createTestTopic(topic, numberOfPartitions, 1); + + final StreamExecutionEnvironment env = + createEnvironment(producerParallelism, timeCharacteristic).enableCheckpointing(500); + + createKafkaShuffle( + env, topic, numElementsPerProducer, producerParallelism, timeCharacteristic, numberOfPartitions) + .map(new FailingIdentityMapper<>(failAfterElements)).setParallelism(1) + .map(new ToInteger(producerParallelism)).setParallelism(1) + .addSink(new ValidatingExactlyOnceSink(numElementsPerProducer * producerParallelism)).setParallelism(1); + + FailingIdentityMapper.failedBefore = false; + + tryExecute(env, topic); + + deleteTestTopic(topic); + } + + /** + * To test failure recovery with partition assignment after processing 2/3 data. + * + *

Schema: (key, timestamp, source instance Id). + * Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3 + */ + private void testAssignedToPartitionFailureRecovery( + int numElementsPerProducer, + TimeCharacteristic timeCharacteristic) throws Exception { + String topic = topic("partition_failure_recovery", timeCharacteristic); + final int numberOfPartitions = 3; + final int producerParallelism = 2; + final int failAfterElements = numElementsPerProducer * producerParallelism * 2 / 3; + + createTestTopic(topic, numberOfPartitions, 1); + + final StreamExecutionEnvironment env = createEnvironment(producerParallelism, timeCharacteristic); + + KeyedStream, Tuple> keyedStream = createKafkaShuffle( + env, + topic, + numElementsPerProducer, + producerParallelism, + timeCharacteristic, + numberOfPartitions); + keyedStream + .process(new PartitionValidator(keyedStream.getKeySelector(), numberOfPartitions, topic)) + .setParallelism(numberOfPartitions) + .map(new ToInteger(producerParallelism)).setParallelism(numberOfPartitions) + .map(new FailingIdentityMapper<>(failAfterElements)).setParallelism(1) + .addSink(new ValidatingExactlyOnceSink(numElementsPerProducer * producerParallelism)).setParallelism(1); + + FailingIdentityMapper.failedBefore = false; + + tryExecute(env, topic); + + deleteTestTopic(topic); + } + + private StreamExecutionEnvironment createEnvironment( + int producerParallelism, + TimeCharacteristic timeCharacteristic) { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(producerParallelism); + env.setStreamTimeCharacteristic(timeCharacteristic); + env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0)); + env.setBufferTimeout(0); + env.enableCheckpointing(500); + + return env; + } + + private static class ToInteger implements MapFunction, Integer> { + private final int producerParallelism; + + ToInteger(int producerParallelism) { + this.producerParallelism = producerParallelism; + } + + @Override + public Integer map(Tuple3 element) throws Exception { + + return element.f0 * producerParallelism + element.f2; + } + } +} diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleITCase.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleITCase.java new file mode 100644 index 0000000000000..805f7ef4358e6 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleITCase.java @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.KeyedStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.connectors.kafka.internal.KafkaShuffleFetcher.KafkaShuffleElement; +import org.apache.flink.streaming.connectors.kafka.internal.KafkaShuffleFetcher.KafkaShuffleElementDeserializer; +import org.apache.flink.streaming.connectors.kafka.internal.KafkaShuffleFetcher.KafkaShuffleRecord; +import org.apache.flink.streaming.connectors.kafka.internal.KafkaShuffleFetcher.KafkaShuffleWatermark; +import org.apache.flink.util.PropertiesUtil; + +import org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableMap; +import org.apache.flink.shaded.guava18.com.google.common.collect.Iterables; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.flink.streaming.api.TimeCharacteristic.EventTime; +import static org.apache.flink.streaming.api.TimeCharacteristic.IngestionTime; +import static org.apache.flink.streaming.api.TimeCharacteristic.ProcessingTime; +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffle.PARTITION_NUMBER; +import static org.apache.flink.streaming.connectors.kafka.shuffle.FlinkKafkaShuffle.PRODUCER_PARALLELISM; +import static org.apache.flink.test.util.TestUtils.tryExecute; +import static org.junit.Assert.fail; + +/** + * Simple End to End Test for Kafka. + */ +public class KafkaShuffleITCase extends KafkaShuffleTestBase { + + @Rule + public final Timeout timeout = Timeout.millis(600000L); + + /** + * To test no data is lost or duplicated end-2-end with the default time characteristic: ProcessingTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testSimpleProcessingTime() throws Exception { + testKafkaShuffle(200000, ProcessingTime); + } + + /** + * To test no data is lost or duplicated end-2-end with time characteristic: IngestionTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testSimpleIngestionTime() throws Exception { + testKafkaShuffle(200000, IngestionTime); + } + + /** + * To test no data is lost or duplicated end-2-end with time characteristic: EventTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testSimpleEventTime() throws Exception { + testKafkaShuffle(100000, EventTime); + } + + /** + * To test data is partitioned to the right partition with time characteristic: ProcessingTime. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testAssignedToPartitionProcessingTime() throws Exception { + testAssignedToPartition(300000, ProcessingTime); + } + + /** + * To test data is partitioned to the right partition with time characteristic: IngestionTime. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testAssignedToPartitionIngestionTime() throws Exception { + testAssignedToPartition(300000, IngestionTime); + } + + /** + * To test data is partitioned to the right partition with time characteristic: EventTime. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testAssignedToPartitionEventTime() throws Exception { + testAssignedToPartition(100000, EventTime); + } + + /** + * To test watermark is monotonically incremental with randomized watermark. + * + *

Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3. + */ + @Test + public void testWatermarkIncremental() throws Exception { + testWatermarkIncremental(100000); + } + + /** + * To test value serialization and deserialization with time characteristic: ProcessingTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testSerDeProcessingTime() throws Exception { + testRecordSerDe(ProcessingTime); + } + + /** + * To test value and watermark serialization and deserialization with time characteristic: IngestionTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testSerDeIngestionTime() throws Exception { + testRecordSerDe(IngestionTime); + } + + /** + * To test value and watermark serialization and deserialization with time characteristic: EventTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testSerDeEventTime() throws Exception { + testRecordSerDe(EventTime); + } + + /** + * To test value and watermark serialization and deserialization with time characteristic: EventTime. + * + *

Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1. + */ + @Test + public void testWatermarkBroadcasting() throws Exception { + final int numberOfPartitions = 3; + final int producerParallelism = 2; + final int numElementsPerProducer = 1000; + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + Map>> results = testKafkaShuffleProducer( + topic("test_watermark_broadcast", EventTime), + env, + numberOfPartitions, + producerParallelism, + numElementsPerProducer, + EventTime); + TypeSerializer> typeSerializer = createTypeSerializer(env); + KafkaShuffleElementDeserializer deserializer = new KafkaShuffleElementDeserializer<>(typeSerializer); + + // Records in a single partition are kept in order + for (int p = 0; p < numberOfPartitions; p++) { + Collection> records = results.get(p); + Map> watermarks = new HashMap<>(); + + for (ConsumerRecord consumerRecord : records) { + Assert.assertNull(consumerRecord.key()); + KafkaShuffleElement element = deserializer.deserialize(consumerRecord); + if (element.isRecord()) { + KafkaShuffleRecord> record = element.asRecord(); + Assert.assertEquals(record.getValue().f1.longValue(), INIT_TIMESTAMP + record.getValue().f0); + Assert.assertEquals(record.getTimestamp().longValue(), record.getValue().f1.longValue()); + } else if (element.isWatermark()) { + KafkaShuffleWatermark watermark = element.asWatermark(); + watermarks.computeIfAbsent(watermark.getSubtask(), k -> new ArrayList<>()); + watermarks.get(watermark.getSubtask()).add(watermark); + } else { + fail("KafkaShuffleElement is either record or watermark"); + } + } + + // According to the setting how watermarks are generated in this ITTest, + // every producer task emits a watermark corresponding to each record + the end-of-event-time watermark. + // Hence each producer sub task generates `numElementsPerProducer + 1` watermarks. + // Each producer sub task broadcasts these `numElementsPerProducer + 1` watermarks to all partitions. + // Thus in total, each producer sub task emits `(numElementsPerProducer + 1) * numberOfPartitions` watermarks. + // From the consumer side, each partition receives `(numElementsPerProducer + 1) * producerParallelism` watermarks, + // with each producer sub task produces `numElementsPerProducer + 1` watermarks. + // Besides, watermarks from the same producer sub task should keep in order. + for (List subTaskWatermarks : watermarks.values()) { + int index = 0; + Assert.assertEquals(numElementsPerProducer + 1, subTaskWatermarks.size()); + for (KafkaShuffleWatermark watermark : subTaskWatermarks) { + if (index == numElementsPerProducer) { + // the last element is the watermark that signifies end-of-event-time + Assert.assertEquals(watermark.getWatermark(), Watermark.MAX_WATERMARK.getTimestamp()); + } else { + Assert.assertEquals(watermark.getWatermark(), INIT_TIMESTAMP + index++); + } + } + } + } + } + + /** + * To test no data is lost or duplicated end-2-end. + * + *

Schema: (key, timestamp, source instance Id). + * Producer Parallelism = 1; Kafka Partition # = 1; Consumer Parallelism = 1 + */ + private void testKafkaShuffle( + int numElementsPerProducer, + TimeCharacteristic timeCharacteristic) throws Exception { + String topic = topic("test_simple", timeCharacteristic); + final int numberOfPartitions = 1; + final int producerParallelism = 1; + + createTestTopic(topic, numberOfPartitions, 1); + + final StreamExecutionEnvironment env = createEnvironment(producerParallelism, timeCharacteristic); + createKafkaShuffle( + env, + topic, + numElementsPerProducer, + producerParallelism, + timeCharacteristic, + numberOfPartitions) + .map(new ElementCountNoMoreThanValidator(numElementsPerProducer * producerParallelism)).setParallelism(1) + .map(new ElementCountNoLessThanValidator(numElementsPerProducer * producerParallelism)).setParallelism(1); + + tryExecute(env, topic); + + deleteTestTopic(topic); + } + + /** + * To test data is partitioned to the right partition. + * + *

Schema: (key, timestamp, source instance Id). + * Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3 + */ + private void testAssignedToPartition( + int numElementsPerProducer, + TimeCharacteristic timeCharacteristic) throws Exception { + String topic = topic("test_assigned_to_partition", timeCharacteristic); + final int numberOfPartitions = 3; + final int producerParallelism = 2; + + createTestTopic(topic, numberOfPartitions, 1); + + final StreamExecutionEnvironment env = createEnvironment(producerParallelism, timeCharacteristic); + + KeyedStream, Tuple> keyedStream = createKafkaShuffle( + env, + topic, + numElementsPerProducer, + producerParallelism, + timeCharacteristic, + numberOfPartitions); + keyedStream + .process(new PartitionValidator(keyedStream.getKeySelector(), numberOfPartitions, topic)) + .setParallelism(numberOfPartitions) + .map(new ElementCountNoMoreThanValidator(numElementsPerProducer * producerParallelism)).setParallelism(1) + .map(new ElementCountNoLessThanValidator(numElementsPerProducer * producerParallelism)).setParallelism(1); + + tryExecute(env, topic); + + deleteTestTopic(topic); + } + + /** + * To watermark from the consumer side always increase. + * + *

Schema: (key, timestamp, source instance Id). + * Producer Parallelism = 2; Kafka Partition # = 3; Consumer Parallelism = 3 + */ + private void testWatermarkIncremental(int numElementsPerProducer) throws Exception { + TimeCharacteristic timeCharacteristic = EventTime; + String topic = topic("test_watermark_incremental", timeCharacteristic); + final int numberOfPartitions = 3; + final int producerParallelism = 2; + + createTestTopic(topic, numberOfPartitions, 1); + + final StreamExecutionEnvironment env = createEnvironment(producerParallelism, timeCharacteristic); + + KeyedStream, Tuple> keyedStream = createKafkaShuffle( + env, + topic, + numElementsPerProducer, + producerParallelism, + timeCharacteristic, + numberOfPartitions, + true); + keyedStream + .process(new WatermarkValidator()) + .setParallelism(numberOfPartitions) + .map(new ElementCountNoMoreThanValidator(numElementsPerProducer * producerParallelism)).setParallelism(1) + .map(new ElementCountNoLessThanValidator(numElementsPerProducer * producerParallelism)).setParallelism(1); + + tryExecute(env, topic); + + deleteTestTopic(topic); + } + + private void testRecordSerDe(TimeCharacteristic timeCharacteristic) throws Exception { + final int numElementsPerProducer = 2000; + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + // Records in a single partition are kept in order + Collection> records = Iterables.getOnlyElement( + testKafkaShuffleProducer( + topic("test_serde", timeCharacteristic), env, 1, 1, numElementsPerProducer, timeCharacteristic).values()); + + switch (timeCharacteristic) { + case ProcessingTime: + // NonTimestampContext, no watermark + Assert.assertEquals(records.size(), numElementsPerProducer); + break; + case IngestionTime: + // IngestionTime uses AutomaticWatermarkContext and it emits a watermark after every `watermarkInterval` + // with default interval 200, hence difficult to control the number of watermarks + break; + case EventTime: + // ManualWatermarkContext + // `numElementsPerProducer` records, `numElementsPerProducer` watermarks, and one end-of-event-time watermark + Assert.assertEquals(records.size(), numElementsPerProducer * 2 + 1); + break; + default: + fail("unknown TimeCharacteristic type"); + } + + TypeSerializer> typeSerializer = createTypeSerializer(env); + + KafkaShuffleElementDeserializer deserializer = new KafkaShuffleElementDeserializer<>(typeSerializer); + + int recordIndex = 0; + int watermarkIndex = 0; + for (ConsumerRecord consumerRecord : records) { + Assert.assertNull(consumerRecord.key()); + KafkaShuffleElement element = deserializer.deserialize(consumerRecord); + if (element.isRecord()) { + KafkaShuffleRecord> record = element.asRecord(); + switch (timeCharacteristic) { + case ProcessingTime: + Assert.assertNull(record.getTimestamp()); + break; + case IngestionTime: + Assert.assertNotNull(record.getTimestamp()); + break; + case EventTime: + Assert.assertEquals(record.getTimestamp().longValue(), record.getValue().f1.longValue()); + break; + default: + fail("unknown TimeCharacteristic type"); + } + Assert.assertEquals(record.getValue().f0.intValue(), recordIndex); + Assert.assertEquals(record.getValue().f1.longValue(), INIT_TIMESTAMP + recordIndex); + Assert.assertEquals(record.getValue().f2.intValue(), 0); + recordIndex++; + } else if (element.isWatermark()) { + switch (timeCharacteristic) { + case ProcessingTime: + fail("Watermarks should not be generated in the case of ProcessingTime"); + break; + case IngestionTime: + break; + case EventTime: + KafkaShuffleWatermark watermark = element.asWatermark(); + Assert.assertEquals(watermark.getSubtask(), 0); + if (watermarkIndex == recordIndex) { + // the last element is the watermark that signifies end-of-event-time + Assert.assertEquals(watermark.getWatermark(), Watermark.MAX_WATERMARK.getTimestamp()); + } else { + Assert.assertEquals(watermark.getWatermark(), INIT_TIMESTAMP + watermarkIndex); + } + break; + default: + fail("unknown TimeCharacteristic type"); + } + watermarkIndex++; + } else { + fail("KafkaShuffleElement is either record or watermark"); + } + } + } + + private Map>> testKafkaShuffleProducer( + String topic, + StreamExecutionEnvironment env, + int numberOfPartitions, + int producerParallelism, + int numElementsPerProducer, + TimeCharacteristic timeCharacteristic) throws Exception { + createTestTopic(topic, numberOfPartitions, 1); + + env.setParallelism(producerParallelism); + env.setRestartStrategy(RestartStrategies.noRestart()); + env.setStreamTimeCharacteristic(timeCharacteristic); + + DataStream> source = + env.addSource(new KafkaSourceFunction(numElementsPerProducer, false)).setParallelism(producerParallelism); + DataStream> input = (timeCharacteristic == EventTime) ? + source.assignTimestampsAndWatermarks(new PunctuatedExtractor()).setParallelism(producerParallelism) : source; + + Properties properties = kafkaServer.getStandardProperties(); + Properties kafkaProperties = PropertiesUtil.flatten(properties); + + kafkaProperties.setProperty(PRODUCER_PARALLELISM, String.valueOf(producerParallelism)); + kafkaProperties.setProperty(PARTITION_NUMBER, String.valueOf(numberOfPartitions)); + kafkaProperties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + kafkaProperties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); + FlinkKafkaShuffle.writeKeyBy(input, topic, kafkaProperties, 0); + + env.execute("Write to " + topic); + ImmutableMap.Builder>> results = ImmutableMap.builder(); + + for (int p = 0; p < numberOfPartitions; p++) { + results.put(p, kafkaServer.getAllRecordsFromTopic(kafkaProperties, topic, p, 5000)); + } + + deleteTestTopic(topic); + + return results.build(); + } + + private StreamExecutionEnvironment createEnvironment( + int producerParallelism, + TimeCharacteristic timeCharacteristic) { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(producerParallelism); + env.setStreamTimeCharacteristic(timeCharacteristic); + env.setRestartStrategy(RestartStrategies.noRestart()); + + return env; + } + + private TypeSerializer> createTypeSerializer(StreamExecutionEnvironment env) { + return new TupleTypeInfo>( + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.LONG_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO) + .createSerializer(env.getConfig()); + } +} diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleTestBase.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleTestBase.java new file mode 100644 index 0000000000000..a42b151a7be75 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/shuffle/KafkaShuffleTestBase.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka.shuffle; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.tuple.Tuple; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.runtime.state.KeyGroupRangeAssignment; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.KeyedStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; +import org.apache.flink.streaming.api.functions.KeyedProcessFunction; +import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; +import org.apache.flink.streaming.connectors.kafka.KafkaConsumerTestBase; +import org.apache.flink.streaming.connectors.kafka.KafkaProducerTestBase; +import org.apache.flink.streaming.connectors.kafka.KafkaTestEnvironmentImpl; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartitionAssigner; +import org.apache.flink.test.util.SuccessException; +import org.apache.flink.util.Collector; + +import org.junit.BeforeClass; + +import java.util.Random; + +import static org.apache.flink.streaming.api.TimeCharacteristic.EventTime; + +/** + * Base Test Class for KafkaShuffle. + */ +public class KafkaShuffleTestBase extends KafkaConsumerTestBase { + static final long INIT_TIMESTAMP = System.currentTimeMillis(); + + @BeforeClass + public static void prepare() throws Exception { + KafkaProducerTestBase.prepare(); + ((KafkaTestEnvironmentImpl) kafkaServer).setProducerSemantic(FlinkKafkaProducer.Semantic.EXACTLY_ONCE); + } + + static class KafkaSourceFunction extends RichParallelSourceFunction> { + private volatile boolean running = true; + private final int numElementsPerProducer; + private final boolean unBounded; + + KafkaSourceFunction(int numElementsPerProducer) { + this.numElementsPerProducer = numElementsPerProducer; + this.unBounded = true; + } + + KafkaSourceFunction(int numElementsPerProducer, boolean unBounded) { + this.numElementsPerProducer = numElementsPerProducer; + this.unBounded = unBounded; + } + + @Override + public void run(SourceContext> ctx) throws Exception{ + long timestamp = INIT_TIMESTAMP; + int sourceInstanceId = getRuntimeContext().getIndexOfThisSubtask(); + for (int i = 0; i < numElementsPerProducer && running; i++) { + ctx.collect(new Tuple3<>(i, timestamp++, sourceInstanceId)); + } + + while (running && unBounded) { + Thread.sleep(100); + } + } + + @Override + public void cancel() { + running = false; + } + } + + static KeyedStream, Tuple> createKafkaShuffle( + StreamExecutionEnvironment env, + String topic, + int numElementsPerProducer, + int producerParallelism, + TimeCharacteristic timeCharacteristic, + int numberOfPartitions) { + return createKafkaShuffle( + env, + topic, + numElementsPerProducer, + producerParallelism, + timeCharacteristic, + numberOfPartitions, + false); + } + + static KeyedStream, Tuple> createKafkaShuffle( + StreamExecutionEnvironment env, + String topic, + int numElementsPerProducer, + int producerParallelism, + TimeCharacteristic timeCharacteristic, + int numberOfPartitions, + boolean randomness) { + DataStream> source = + env.addSource(new KafkaSourceFunction(numElementsPerProducer)).setParallelism(producerParallelism); + DataStream> input = (timeCharacteristic == EventTime) ? + source.assignTimestampsAndWatermarks(new PunctuatedExtractor(randomness)).setParallelism(producerParallelism) : source; + + return FlinkKafkaShuffle.persistentKeyBy( + input, + topic, + producerParallelism, + numberOfPartitions, + kafkaServer.getStandardProperties(), + 0); + } + + static class PunctuatedExtractor implements AssignerWithPunctuatedWatermarks> { + private static final long serialVersionUID = 1L; + boolean randomness; + Random rnd = new Random(123); + + PunctuatedExtractor() { + randomness = false; + } + + PunctuatedExtractor(boolean randomness) { + this.randomness = randomness; + } + + @Override + public long extractTimestamp(Tuple3 element, long previousTimestamp) { + return element.f1; + } + + @Override + public Watermark checkAndGetNextWatermark(Tuple3 lastElement, long extractedTimestamp) { + long randomValue = randomness ? rnd.nextInt(10) : 0; + return new Watermark(extractedTimestamp + randomValue); + } + } + + static class PartitionValidator + extends KeyedProcessFunction, Tuple3> { + private final KeySelector, Tuple> keySelector; + private final int numberOfPartitions; + private final String topic; + + private int previousPartition; + + PartitionValidator( + KeySelector, Tuple> keySelector, + int numberOfPartitions, + String topic) { + this.keySelector = keySelector; + this.numberOfPartitions = numberOfPartitions; + this.topic = topic; + this.previousPartition = -1; + } + + @Override + public void processElement( + Tuple3 in, + Context ctx, + Collector> out) throws Exception { + int expectedPartition = KeyGroupRangeAssignment + .assignKeyToParallelOperator(keySelector.getKey(in), numberOfPartitions, numberOfPartitions); + int indexOfThisSubtask = getRuntimeContext().getIndexOfThisSubtask(); + KafkaTopicPartition partition = new KafkaTopicPartition(topic, expectedPartition); + + // This is how Kafka assign partition to subTask; + boolean rightAssignment = + KafkaTopicPartitionAssigner.assign(partition, numberOfPartitions) == indexOfThisSubtask; + boolean samePartition = (previousPartition == expectedPartition) || (previousPartition == -1); + previousPartition = expectedPartition; + + if (!(rightAssignment && samePartition)) { + throw new Exception("Error: Kafka partition assignment error "); + } + out.collect(in); + } + } + + static class WatermarkValidator + extends KeyedProcessFunction, Tuple3> { + private long previousWatermark = Long.MIN_VALUE; // initial watermark get from timeService + + @Override + public void processElement( + Tuple3 in, + Context ctx, + Collector> out) throws Exception { + + long watermark = ctx.timerService().currentWatermark(); + + // Notice that the timerService might not be updated if no new watermark has been emitted, hence equivalent + // watermark is allowed, strictly incremental check is done when fetching watermark from KafkaShuffleFetcher. + if (watermark < previousWatermark) { + throw new Exception( + "Error: watermark should always increase. current watermark : previous watermark [" + + watermark + " : " + previousWatermark + "]"); + } + previousWatermark = watermark; + + out.collect(in); + } + } + + static class ElementCountNoLessThanValidator + implements MapFunction, Tuple3> { + private final int totalCount; + private int counter = 0; + + ElementCountNoLessThanValidator(int totalCount) { + this.totalCount = totalCount; + } + + @Override + public Tuple3 map(Tuple3 element) throws Exception { + counter++; + + if (counter == totalCount) { + throw new SuccessException(); + } + + return element; + } + } + + static class ElementCountNoMoreThanValidator + implements MapFunction, Tuple3> { + private final int totalCount; + private int counter = 0; + + ElementCountNoMoreThanValidator(int totalCount) { + this.totalCount = totalCount; + } + + @Override + public Tuple3 map(Tuple3 element) throws Exception { + counter++; + + if (counter > totalCount) { + throw new Exception("Error: number of elements more than expected"); + } + + return element; + } + } + + String topic(String prefix, TimeCharacteristic timeCharacteristic) { + return prefix + "_" + timeCharacteristic; + } +} diff --git a/flink-connectors/flink-connector-kafka/src/test/resources/log4j2-test.properties b/flink-connectors/flink-connector-kafka/src/test/resources/log4j2-test.properties new file mode 100644 index 0000000000000..863665cf4f751 --- /dev/null +++ b/flink-connectors/flink-connector-kafka/src/test/resources/log4j2-test.properties @@ -0,0 +1,38 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level = OFF +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.target = SYSTEM_ERR +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n + +logger.kafka.name = kafka +logger.kafka.level = OFF +logger.kafka2.name = state.change +logger.kafka2.level = OFF + +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = OFF +logger.I0Itec.name = org.I0Itec +logger.I0Itec.level = OFF From d8a77cbf93007bf970963a4499aa06501c0d9808 Mon Sep 17 00:00:00 2001 From: Gary Yao Date: Mon, 18 May 2020 09:17:58 +0200 Subject: [PATCH 007/773] [FLINK-17792][tests] Catch and log exception if jstack fails jstack can fail if the JVM process that we want to sample exits while or before we invoke jstack. Since a JVM process is free to exit at any time, we should not propagate the exception so that we do not fail the test prematurely. --- flink-jepsen/src/jepsen/flink/utils.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flink-jepsen/src/jepsen/flink/utils.clj b/flink-jepsen/src/jepsen/flink/utils.clj index 8f6f6545a76e9..5d8e7126fa654 100644 --- a/flink-jepsen/src/jepsen/flink/utils.clj +++ b/flink-jepsen/src/jepsen/flink/utils.clj @@ -133,7 +133,10 @@ (defn- write-jstack! [pid out-path] - (c/exec :jstack :-l pid :> out-path)) + (try + (c/exec :jstack :-l pid :> out-path) + (catch Exception e + (warn e "Failed to invoke jstack on pid" pid)))) (defn dump-jstack-by-pattern! "Dumps the output of jstack for all JVMs that match one of the specified patterns." From a2deff2967b7de423b10f7f01a41c06565c37e62 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 18 May 2020 09:37:07 +0200 Subject: [PATCH 008/773] [minor] Allow relative paths in LocalFileSystem --- .../org/apache/flink/core/fs/local/LocalFileSystem.java | 8 ++------ .../flink/core/fs/local/LocalRecoverableSerializer.java | 4 ++-- .../flink/core/fs/local/LocalRecoverableWriter.java | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java index 694655828acca..1a068762a1d08 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalFileSystem.java @@ -49,6 +49,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -308,14 +309,9 @@ public FileSystemKind getKind() { /** * Converts the given Path to a File for this file system. - * - *

If the path is not absolute, it is interpreted relative to this FileSystem's working directory. */ public File pathToFile(Path path) { - if (!path.isAbsolute()) { - path = new Path(getWorkingDirectory(), path); - } - return new File(path.toUri().getPath()); + return Paths.get(path.getPath()).toFile(); } // ------------------------------------------------------------------------ diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableSerializer.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableSerializer.java index 685f7c17abed8..8a6892899f462 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableSerializer.java @@ -52,8 +52,8 @@ public int getVersion() { @Override public byte[] serialize(LocalRecoverable obj) throws IOException { - final byte[] targetFileBytes = obj.targetFile().getAbsolutePath().getBytes(CHARSET); - final byte[] tempFileBytes = obj.tempFile().getAbsolutePath().getBytes(CHARSET); + final byte[] targetFileBytes = obj.targetFile().toString().getBytes(CHARSET); + final byte[] tempFileBytes = obj.tempFile().toString().getBytes(CHARSET); final byte[] targetBytes = new byte[20 + targetFileBytes.length + tempFileBytes.length]; ByteBuffer bb = ByteBuffer.wrap(targetBytes).order(ByteOrder.LITTLE_ENDIAN); diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java index a43e0b6b6bf4e..bae73149b28f3 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java @@ -116,7 +116,6 @@ public boolean supportsResume() { @VisibleForTesting static File generateStagingTempFilePath(File targetFile) { - checkArgument(targetFile.isAbsolute(), "targetFile must be absolute"); checkArgument(!targetFile.isDirectory(), "targetFile must not be a directory"); final File parent = targetFile.getParentFile(); From b8fab2ffdc8f07b4fb7043dd96b25633f8e2eed9 Mon Sep 17 00:00:00 2001 From: GuoWei Ma Date: Mon, 18 May 2020 15:19:07 +0800 Subject: [PATCH 009/773] [FLINK-17593][Connectors/FileSystem] Turn BucketStateSerializerTest into an upgrade test --- .../filesystem/BucketStateSerializerTest.java | 424 +++++++++++------- .../empty-v1/snapshot | Bin 0 -> 128 bytes ...gress.a88d5993-77bc-44ce-880b-9f2a43b59ab4 | 2 + ...gress.7c0f2bd7-3078-48e8-9af2-d8773fb949c5 | 2 + ...gress.6729a640-0585-4785-a652-89802950c663 | 2 + ...gress.b4bcb0e9-5c9e-45dd-8963-1b163343544d | 2 + ...gress.e1e9e48d-0db6-4dd7-8a4d-fb4ebe7ed8ac | 2 + .../full-no-in-progress-v1-template/snapshot | Bin 0 -> 1537 bytes ...gress.8fec17e9-5d54-4fa9-aebb-70736fe03c82 | 2 + ...gress.0035b171-2759-403a-8d6c-4612b28a7a6c | 2 + ...gress.49da8048-af6b-4665-b4f6-b659cb38dc97 | 2 + ...gress.d13ec4e0-07b5-4f4e-9be8-9fb457cbcde9 | 2 + ...gress.123ac2c7-f92a-476a-a848-1369b93d82a7 | 2 + ...gress.32f5a28f-20e1-48da-9951-10e795133d64 | 1 + .../full-v1-template/snapshot | Bin 0 -> 1613 bytes ...gress.a70190d6-d080-43a8-b414-746b09d3a8a0 | 1 + .../only-in-progress-v1/snapshot | Bin 0 -> 404 bytes 17 files changed, 282 insertions(+), 164 deletions(-) create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v1/snapshot create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-0.inprogress.a88d5993-77bc-44ce-880b-9f2a43b59ab4 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-1.inprogress.7c0f2bd7-3078-48e8-9af2-d8773fb949c5 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-2.inprogress.6729a640-0585-4785-a652-89802950c663 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-3.inprogress.b4bcb0e9-5c9e-45dd-8963-1b163343544d create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-4.inprogress.e1e9e48d-0db6-4dd7-8a4d-fb4ebe7ed8ac create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/snapshot create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-0.inprogress.8fec17e9-5d54-4fa9-aebb-70736fe03c82 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-1.inprogress.0035b171-2759-403a-8d6c-4612b28a7a6c create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-2.inprogress.49da8048-af6b-4665-b4f6-b659cb38dc97 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-3.inprogress.d13ec4e0-07b5-4f4e-9be8-9fb457cbcde9 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-4.inprogress.123ac2c7-f92a-476a-a848-1369b93d82a7 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-5.inprogress.32f5a28f-20e1-48da-9951-10e795133d64 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v1-template/snapshot create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v1/bucket/test-bucket/.part-0-0.inprogress.a70190d6-d080-43a8-b414-746b09d3a8a0 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v1/snapshot diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java index f2c1f8b2885dd..81c57663c89c0 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java @@ -18,253 +18,349 @@ package org.apache.flink.streaming.api.functions.sink.filesystem; -import org.apache.flink.core.fs.FileStatus; +import org.apache.flink.api.common.serialization.SimpleStringEncoder; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableFsDataOutputStream; import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerialization; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; +import org.apache.flink.util.FileUtils; import org.junit.Assert; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; -import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.junit.Assert.assertThat; /** - * Tests for the {@link BucketStateSerializer}. + * Tests for the {@link BucketStateSerializer} that verify we can still read snapshots written using + * an older version of the serializer. We keep snapshots for all previous versions in version + * control (including the current version). The tests verify that the current version of the + * serializer can still read data from all previous versions. */ +@RunWith(Parameterized.class) public class BucketStateSerializerTest { + private static final int CURRENT_VERSION = 1; + + @Parameterized.Parameters(name = "Previous Version = {0}") + public static Collection previousVersions() { + return Arrays.asList(1); + } + + @Parameterized.Parameter + public Integer previousVersion; + private static final String IN_PROGRESS_CONTENT = "writing"; private static final String PENDING_CONTENT = "wrote"; + private static final String BUCKET_ID = "test-bucket"; + @ClassRule public static TemporaryFolder tempFolder = new TemporaryFolder(); + private static java.nio.file.Path getResourcePath( + String scenarioName, + int version) { + return Paths.get("src/test/resources/") + .resolve("bucket-state-migration-test") + .resolve(scenarioName + "-v" + version); + } + + private static java.nio.file.Path getSnapshotPath( + String scenarioName, + int version) { + java.nio.file.Path basePath = getResourcePath(scenarioName, version); + return basePath.resolve("snapshot"); + } + + private static java.nio.file.Path getOutputPath(String scenarioName, int version) { + java.nio.file.Path basePath = getResourcePath(scenarioName, version); + return basePath.resolve("bucket"); + } + @Test - public void testSerializationEmpty() throws IOException { - final File testFolder = tempFolder.newFolder(); - final FileSystem fs = FileSystem.get(testFolder.toURI()); - final RecoverableWriter writer = fs.createRecoverableWriter(); + @Ignore + public void prepareDeserializationEmpty() throws IOException { - final Path testBucket = new Path(testFolder.getPath(), "test"); + final String scenarioName = "empty"; + final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION); - final BucketState bucketState = new BucketState<>( - "test", testBucket, Long.MAX_VALUE, null, new HashMap<>()); + FileUtils.deleteDirectory(scenarioPath.toFile()); + Files.createDirectories(scenarioPath); - final SimpleVersionedSerializer> serializer = - new BucketStateSerializer<>( - writer.getResumeRecoverableSerializer(), - writer.getCommitRecoverableSerializer(), - SimpleVersionedStringSerializer.INSTANCE - ); + final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION); + final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString()); - byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); - final BucketState recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); + final Bucket bucket = + createNewBucket(testBucketPath); - Assert.assertEquals(testBucket, recoveredState.getBucketPath()); - Assert.assertNull(recoveredState.getInProgressResumableFile()); - Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty()); + final BucketState bucketState = bucket.onReceptionOfCheckpoint(0); + + byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize( + bucketStateSerializer(), + bucketState); + Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes); } @Test - public void testSerializationOnlyInProgress() throws IOException { - final File testFolder = tempFolder.newFolder(); - final FileSystem fs = FileSystem.get(testFolder.toURI()); + public void testSerializationEmpty() throws IOException { - final Path testBucket = new Path(testFolder.getPath(), "test"); + final String scenarioName = "empty"; + final java.nio.file.Path outputPath = getOutputPath(scenarioName, previousVersion); + final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString()); + final BucketState recoveredState = readBucketState(scenarioName, previousVersion); - final RecoverableWriter writer = fs.createRecoverableWriter(); - final RecoverableFsDataOutputStream stream = writer.open(testBucket); - stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); + final Bucket bucket = restoreBucket(0, recoveredState); - final RecoverableWriter.ResumeRecoverable current = stream.persist(); + Assert.assertEquals(testBucketPath, bucket.getBucketPath()); + Assert.assertNull(bucket.getInProgressPart()); + Assert.assertTrue(bucket.getPendingPartsPerCheckpoint().isEmpty()); + } - final BucketState bucketState = new BucketState<>( - "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); + @Test + @Ignore + public void prepareDeserializationOnlyInProgress() throws IOException { + + final String scenarioName = "only-in-progress"; + final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION); + FileUtils.deleteDirectory(scenarioPath.toFile()); + Files.createDirectories(scenarioPath); - final SimpleVersionedSerializer> serializer = - new BucketStateSerializer<>( - writer.getResumeRecoverableSerializer(), - writer.getCommitRecoverableSerializer(), - SimpleVersionedStringSerializer.INSTANCE - ); + final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION); + final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString()); - final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); + final Bucket bucket = + createNewBucket(testBucketPath); - // to simulate that everything is over for file. - stream.close(); + bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis()); - final BucketState recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); + final BucketState bucketState = bucket.onReceptionOfCheckpoint(0); - Assert.assertEquals(testBucket, recoveredState.getBucketPath()); + final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize( + bucketStateSerializer(), bucketState); - FileStatus[] statuses = fs.listStatus(testBucket.getParent()); - Assert.assertEquals(1L, statuses.length); - Assert.assertTrue( - statuses[0].getPath().getPath().startsWith( - (new Path(testBucket.getParent(), ".test.inprogress")).getPath()) - ); + Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes); } @Test - public void testSerializationFull() throws IOException { - final int noOfTasks = 5; + public void testSerializationOnlyInProgress() throws IOException { - final File testFolder = tempFolder.newFolder(); - final FileSystem fs = FileSystem.get(testFolder.toURI()); - final RecoverableWriter writer = fs.createRecoverableWriter(); + final String scenarioName = "only-in-progress"; + final java.nio.file.Path outputPath = getOutputPath(scenarioName, previousVersion); - final Path bucketPath = new Path(testFolder.getPath()); + final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString()); - // pending for checkpoints - final Map> commitRecoverables = new HashMap<>(); - for (int i = 0; i < noOfTasks; i++) { - final List recoverables = new ArrayList<>(); - for (int j = 0; j < 2 + i; j++) { - final Path part = new Path(bucketPath, "part-" + i + '-' + j); - - final RecoverableFsDataOutputStream stream = writer.open(part); - stream.write((PENDING_CONTENT + '-' + j).getBytes(Charset.forName("UTF-8"))); - recoverables.add(stream.closeForCommit().getRecoverable()); - } - commitRecoverables.put((long) i, recoverables); - } + final BucketState recoveredState = readBucketState(scenarioName, previousVersion); - // in-progress - final Path testBucket = new Path(bucketPath, "test-2"); - final RecoverableFsDataOutputStream stream = writer.open(testBucket); - stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); + final Bucket bucket = restoreBucket(0, recoveredState); - final RecoverableWriter.ResumeRecoverable current = stream.persist(); + Assert.assertEquals(testBucketPath, bucket.getBucketPath()); - final BucketState bucketState = new BucketState<>( - "test-2", bucketPath, Long.MAX_VALUE, current, commitRecoverables); - final SimpleVersionedSerializer> serializer = - new BucketStateSerializer<>( - writer.getResumeRecoverableSerializer(), - writer.getCommitRecoverableSerializer(), - SimpleVersionedStringSerializer.INSTANCE - ); - stream.close(); + //check restore the correct in progress file writer + Assert.assertEquals(8, bucket.getInProgressPart().getSize()); - byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); + long numFiles = Files.list(Paths.get(testBucketPath.toString())) + .map(file -> { + assertThat( + file.getFileName().toString(), + startsWith(".part-0-0.inprogress")); + return 1; + }) + .count(); - final BucketState recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); + assertThat(numFiles, is(1L)); + } - Assert.assertEquals(bucketPath, recoveredState.getBucketPath()); + @Test + @Ignore + public void prepareDeserializationFull() throws IOException { + prepareDeserializationFull(true, "full"); + } - final Map> recoveredRecoverables = recoveredState.getCommittableFilesPerCheckpoint(); - Assert.assertEquals(5L, recoveredRecoverables.size()); + @Test + public void testSerializationFull() throws IOException { + testDeserializationFull(true, "full"); + } - // recover and commit - for (Map.Entry> entry: recoveredRecoverables.entrySet()) { - for (RecoverableWriter.CommitRecoverable recoverable: entry.getValue()) { - writer.recoverForCommit(recoverable).commit(); - } - } + @Test + @Ignore + public void prepareDeserializationNullInProgress() throws IOException { + prepareDeserializationFull(false, "full-no-in-progress"); + } - FileStatus[] filestatuses = fs.listStatus(bucketPath); - Set paths = new HashSet<>(filestatuses.length); - for (FileStatus filestatus : filestatuses) { - paths.add(filestatus.getPath().getPath()); - } + @Test + public void testSerializationNullInProgress() throws IOException { + testDeserializationFull(false, "full-no-in-progress"); + } - for (int i = 0; i < noOfTasks; i++) { - for (int j = 0; j < 2 + i; j++) { - final String part = new Path(bucketPath, "part-" + i + '-' + j).getPath(); - Assert.assertTrue(paths.contains(part)); - paths.remove(part); - } - } + private static void prepareDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException { - // only the in-progress must be left - Assert.assertEquals(1L, paths.size()); + final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION); + FileUtils.deleteDirectory(Paths.get(scenarioPath.toString() + "-template").toFile()); + Files.createDirectories(scenarioPath); - // verify that the in-progress file is still there - Assert.assertTrue(paths.iterator().next().startsWith( - (new Path(testBucket.getParent(), ".test-2.inprogress").getPath()))); - } + final int noOfPendingCheckpoints = 5; - @Test - public void testSerializationNullInProgress() throws IOException { - final int noOfTasks = 5; + final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION); - final File testFolder = tempFolder.newFolder(); - final FileSystem fs = FileSystem.get(testFolder.toURI()); - final RecoverableWriter writer = fs.createRecoverableWriter(); + final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString()); - final Path bucketPath = new Path(testFolder.getPath()); + final Bucket bucket = createNewBucket(testBucketPath); + BucketState bucketState = null; // pending for checkpoints - final Map> commitRecoverables = new HashMap<>(); - for (int i = 0; i < noOfTasks; i++) { - final List recoverables = new ArrayList<>(); - for (int j = 0; j < 2 + i; j++) { - final Path part = new Path(bucketPath, "test-" + i + '-' + j); - - final RecoverableFsDataOutputStream stream = writer.open(part); - stream.write((PENDING_CONTENT + '-' + j).getBytes(Charset.forName("UTF-8"))); - recoverables.add(stream.closeForCommit().getRecoverable()); - } - commitRecoverables.put((long) i, recoverables); + for (int i = 0; i < noOfPendingCheckpoints; i++) { + // write 10 bytes to the in progress file + bucket.write(PENDING_CONTENT, System.currentTimeMillis()); + bucket.write(PENDING_CONTENT, System.currentTimeMillis()); + // every checkpoint would produce a pending file + bucketState = bucket.onReceptionOfCheckpoint(i); + } + + if (withInProgress) { + // create a in progress file + bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis()); + + // 5 pending files and 1 in progress file + bucketState = bucket.onReceptionOfCheckpoint(noOfPendingCheckpoints); } - final RecoverableWriter.ResumeRecoverable current = null; + final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(bucketStateSerializer(), bucketState); - final BucketState bucketState = new BucketState<>( - "", bucketPath, Long.MAX_VALUE, current, commitRecoverables); + Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes); - final SimpleVersionedSerializer> serializer = new BucketStateSerializer<>( - writer.getResumeRecoverableSerializer(), - writer.getCommitRecoverableSerializer(), - SimpleVersionedStringSerializer.INSTANCE - ); + // copy the scenario file to a template directory. + // it is because that the test `testSerializationFull` would change the in progress file to pending files. + moveToTemplateDirectory(scenarioPath); + } - byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); + private void testDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException { - final BucketState recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); + try { + final java.nio.file.Path outputPath = getOutputPath(scenarioName, previousVersion); + final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString()); + // restore the state + final BucketState recoveredState = readBucketStateFromTemplate(scenarioName, previousVersion); + final int noOfPendingCheckpoints = 5; - Assert.assertEquals(bucketPath, recoveredState.getBucketPath()); - Assert.assertNull(recoveredState.getInProgressResumableFile()); + // there are 5 checkpoint does not complete. + final Map> + pendingFileRecoverables = recoveredState.getCommittableFilesPerCheckpoint(); + Assert.assertEquals(5L, pendingFileRecoverables.size()); - final Map> recoveredRecoverables = recoveredState.getCommittableFilesPerCheckpoint(); - Assert.assertEquals(5L, recoveredRecoverables.size()); + final Set beforeRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)) + .map(file -> file.getFileName().toString()) + .collect(Collectors.toSet()); - // recover and commit - for (Map.Entry> entry: recoveredRecoverables.entrySet()) { - for (RecoverableWriter.CommitRecoverable recoverable: entry.getValue()) { - writer.recoverForCommit(recoverable).commit(); + // before retsoring all file has "inprogress" + for (int i = 0; i < noOfPendingCheckpoints; i++) { + final String part = ".part-0-" + i + ".inprogress"; + assertThat(beforeRestorePaths, hasItem(startsWith(part))); } - } - FileStatus[] filestatuses = fs.listStatus(bucketPath); - Set paths = new HashSet<>(filestatuses.length); - for (FileStatus filestatus : filestatuses) { - paths.add(filestatus.getPath().getPath()); - } + // recover and commit + final Bucket bucket = restoreBucket(noOfPendingCheckpoints + 1, recoveredState); + Assert.assertEquals(testBucketPath, bucket.getBucketPath()); + Assert.assertEquals(0, bucket.getPendingPartsPerCheckpoint().size()); + + final Set afterRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)) + .map(file -> file.getFileName().toString()) + .collect(Collectors.toSet()); + + // after restoring all pending files are comitted. + // there is no "inporgress" in file name for the committed files. + for (int i = 0; i < noOfPendingCheckpoints; i++) { + final String part = "part-0-" + i; + assertThat(afterRestorePaths, hasItem(part)); + afterRestorePaths.remove(part); + } + + if (withInProgress) { + // only the in-progress must be left + assertThat(afterRestorePaths, iterableWithSize(1)); - for (int i = 0; i < noOfTasks; i++) { - for (int j = 0; j < 2 + i; j++) { - final String part = new Path(bucketPath, "test-" + i + '-' + j).getPath(); - Assert.assertTrue(paths.contains(part)); - paths.remove(part); + // verify that the in-progress file is still there + assertThat(afterRestorePaths, hasItem(startsWith(".part-0-" + noOfPendingCheckpoints + ".inprogress"))); + } else { + assertThat(afterRestorePaths, empty()); } + } finally { + FileUtils.deleteDirectory(getResourcePath(scenarioName, previousVersion).toFile()); } + } + + private static Bucket createNewBucket(final Path bucketPath) throws IOException { + return Bucket.getNew( + FileSystem.getLocalFileSystem().createRecoverableWriter(), + 0, + BUCKET_ID, + bucketPath, + 0, + new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + DefaultRollingPolicy.builder().withMaxPartSize(10).build(), + OutputFileConfig.builder().build()); + } + + private static Bucket restoreBucket(final int initialPartCounter, final BucketState bucketState) throws IOException { + return Bucket.restore( + FileSystem.getLocalFileSystem().createRecoverableWriter(), + 0, + initialPartCounter, + new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + DefaultRollingPolicy.builder().withMaxPartSize(10).build(), + bucketState, + OutputFileConfig.builder().build()); + } + + private static BucketState readBucketState(final String scenarioName, final int version) throws IOException { + byte[] bytes = Files.readAllBytes(getSnapshotPath(scenarioName, version)); + return SimpleVersionedSerialization.readVersionAndDeSerialize(bucketStateSerializer(), bytes); + } + + private static BucketState readBucketStateFromTemplate(final String scenarioName, final int version) throws IOException { + final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, version); + + // clear the scenario files first + FileUtils.deleteDirectory(scenarioPath.toFile()); + + // prepare the scenario files + FileUtils.copy(new Path(scenarioPath.toString() + "-template"), new Path(scenarioPath.toString()), false); + + return readBucketState(scenarioName, version); + } + + private static SimpleVersionedSerializer> bucketStateSerializer() throws IOException { + RecoverableWriter recoverableWriter = FileSystem.getLocalFileSystem().createRecoverableWriter(); + return new BucketStateSerializer<>( + recoverableWriter.getResumeRecoverableSerializer(), + recoverableWriter.getCommitRecoverableSerializer(), + SimpleVersionedStringSerializer.INSTANCE); + } - // only the in-progress must be left - Assert.assertTrue(paths.isEmpty()); + private static void moveToTemplateDirectory(java.nio.file.Path scenarioPath) throws IOException { + FileUtils.copy(new Path(scenarioPath.toString()), new Path(scenarioPath.toString() + "-template"), false); + FileUtils.deleteDirectory(scenarioPath.toFile()); } } diff --git a/flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v1/snapshot b/flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v1/snapshot new file mode 100644 index 0000000000000000000000000000000000000000..9700f3d8932fc9fc6e2c94b93cfcf04b5c9cc933 GIT binary patch literal 128 zcmZQzU|?imV5pEQ^R5KaAT~cYkSIwlF40XYP0mg&Vel$0O4bJn=oh6H=a&{Grxxo& vq;!i*5=&Bbb2HP65=%1k^K`+gQgaJRDs{^Yp^Cs-p=QzPL_7pKOZoQSUpO z>b8N)SJmlofTo4Ifn(S1^P(Qk<+$Xs(YW*1@3rN6!zsvmNXC}mSC6l|w0Rx}Y>wag znd>mc(RK+a(Af%XZxyz6bA#Dht%ERzgHp}|qK}|8bzmxHkvn0m!+(;2X&IJsc9wF= zqCBmb3ko<@8c;_K##Ri1R*F~7pz&g4xyx}e7B)Kzo3cp7jFp)GQlSN)%AJ*hfi{{l zBdC{>k1Y2~vhdki_>{$=^NvOX!kY*vLdYVLL+MJ%IdXv*MwW*qS#Wk1oU+6+nut08 V4Nd}ve4Vt#0F}e&qKZLV{|7$F?`_Dx_ka0`<&d}l%!2f3w|5kQcIqiI#k_3Ri|dXtwU<6-z#WZY!kdM z<~6p9^$PY?dAnS~j`cPBmxoqlQs1v0zRJjMEct$YZl~8J>FlNWXYtXdRU52ZprFR< zI6JhQEmoU)JuiYZGr_Y$ZZmM27)TotoKp-KO{PR4qLziL1HLSNus;EA`;(&yK*tw= z4h66|1!j^16_o^;(E%p!!B8V~PE-WTd&Ym8KRP%YADj)rDHX~y!+;y*KvIEVqYfZ7 zbd+eDI4@D7Duxno+7;4lO8dGqGO z7>h{X@l?$$`0TJJ2en72$L67tUA=4glHg8zJGw@D{wsk;g&JOc=M?tVKtNgZoS`=` zINMQ&hbjHz1ItRwB6qUK3ylveM7Ij#9R-Wr$v$M8Oy+}f2m-`5?8!=>^k#iQqH>Xz dqRpTcg#dY01=KQ?kjtzVWlOOVbno&E_64 Date: Wed, 13 May 2020 21:15:03 +0800 Subject: [PATCH 010/773] [FLINK-17593][Connectors/FileSystem] Support arbitrary recovery mechanism for PartFileWriter This change includes two things: 1. Make the PartFileWriter generic and decouple the PartFileWriter and RecoverableStream. According to different pre-commit / commit methods, this change allows us to extend different types of PartFileWriter. 2. Make the Bucket/Buckets depends on the PartFileFactory instead of RecoverableWriter. --- .../flink/core/fs/RecoverableWriter.java | 6 +- .../core/fs/local/LocalRecoverableWriter.java | 2 +- .../fs/hdfs/HadoopRecoverableWriter.java | 2 +- .../filesystem/AbstractPartFileWriter.java | 58 ++++ .../api/functions/sink/filesystem/Bucket.java | 166 ++++------ .../sink/filesystem/BucketFactory.java | 7 +- .../sink/filesystem/BucketState.java | 32 +- .../filesystem/BucketStateSerializer.java | 146 ++++++--- .../sink/filesystem/BucketWriter.java | 109 +++++++ .../functions/sink/filesystem/Buckets.java | 37 +-- .../sink/filesystem/BulkBucketWriter.java | 72 +++++ .../sink/filesystem/BulkPartWriter.java | 56 +--- .../filesystem/DefaultBucketFactoryImpl.java | 13 +- .../sink/filesystem/InProgressFileWriter.java | 70 +++++ .../OutputStreamBasedPartFileWriter.java | 296 ++++++++++++++++++ .../sink/filesystem/PartFileWriter.java | 141 --------- .../sink/filesystem/RowWiseBucketWriter.java | 68 ++++ .../sink/filesystem/RowWisePartWriter.java | 50 +-- .../sink/filesystem/StreamingFileSink.java | 4 +- .../sink/filesystem/WriterProperties.java | 67 ++++ .../filesystem/BucketAssignerITCases.java | 3 +- .../filesystem/BucketStateSerializerTest.java | 35 ++- .../functions/sink/filesystem/BucketTest.java | 58 ++-- .../sink/filesystem/BucketsTest.java | 19 +- .../sink/filesystem/RollingPolicyTest.java | 3 +- .../functions/sink/filesystem/TestUtils.java | 7 +- .../utils/NoOpRecoverableWriter.java | 2 +- 27 files changed, 1033 insertions(+), 496 deletions(-) create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/AbstractPartFileWriter.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketWriter.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkBucketWriter.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/InProgressFileWriter.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputStreamBasedPartFileWriter.java delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWiseBucketWriter.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/WriterProperties.java diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java b/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java index 7d54b11bbbc82..b92da886ea990 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/RecoverableWriter.java @@ -138,10 +138,8 @@ public interface RecoverableWriter { * recover from a (potential) failure. These can be temporary files that were written * to the filesystem or objects that were uploaded to S3. * - *

NOTE: This operation should not throw an exception if the resumable has already - * been cleaned up and the resources have been freed. But the contract is that it will throw - * an {@link UnsupportedOperationException} if it is called for a {@code RecoverableWriter} - * whose {@link #requiresCleanupOfRecoverableState()} returns {@code false}. + *

NOTE: This operation should not throw an exception, but return false if the cleanup did not + * happen for any reason. * * @param resumable The {@link ResumeRecoverable} whose state we want to clean-up. * @return {@code true} if the resources were successfully freed, {@code false} otherwise diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java index bae73149b28f3..2a97b85981c44 100644 --- a/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/local/LocalRecoverableWriter.java @@ -77,7 +77,7 @@ public boolean requiresCleanupOfRecoverableState() { @Override public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { - throw new UnsupportedOperationException(); + return false; } @Override diff --git a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java index d325f2cfc5665..91d76c6596ff3 100644 --- a/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java +++ b/flink-filesystems/flink-hadoop-fs/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopRecoverableWriter.java @@ -95,7 +95,7 @@ public boolean requiresCleanupOfRecoverableState() { @Override public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { - throw new UnsupportedOperationException(); + return false; } @Override diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/AbstractPartFileWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/AbstractPartFileWriter.java new file mode 100644 index 0000000000000..0350a8ff0671c --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/AbstractPartFileWriter.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +/** + * An abstract writer for the currently open part file in a specific {@link Bucket}. + * @param the element type. + * @param the bucket id type. + */ +public abstract class AbstractPartFileWriter implements InProgressFileWriter { + + private final BucketID bucketID; + + private final long creationTime; + + private long lastUpdateTime; + + public AbstractPartFileWriter(final BucketID bucketID, final long createTime) { + this.bucketID = bucketID; + this.creationTime = createTime; + this.lastUpdateTime = createTime; + } + + @Override + public BucketID getBucketId() { + return bucketID; + } + + @Override + public long getCreationTime() { + return creationTime; + } + + @Override + public long getLastUpdateTime() { + return lastUpdateTime; + } + + void markWrite(long now) { + this.lastUpdateTime = now; + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java index e7abd3793c5ae..5e9a72b0c25f8 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Bucket.java @@ -21,10 +21,6 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableFsDataOutputStream; -import org.apache.flink.core.fs.RecoverableWriter; -import org.apache.flink.core.fs.RecoverableWriter.CommitRecoverable; -import org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,48 +56,44 @@ public class Bucket { private final int subtaskIndex; - private final PartFileWriter.PartFileFactory partFileFactory; - - private final RecoverableWriter fsWriter; + private final BucketWriter bucketWriter; private final RollingPolicy rollingPolicy; - private final NavigableMap resumablesPerCheckpoint; + private final NavigableMap inProgressFileRecoverablesPerCheckpoint; - private final NavigableMap> pendingPartsPerCheckpoint; + private final NavigableMap> pendingFileRecoverablesPerCheckpoint; private final OutputFileConfig outputFileConfig; private long partCounter; @Nullable - private PartFileWriter inProgressPart; + private InProgressFileWriter inProgressPart; - private List pendingPartsForCurrentCheckpoint; + private List pendingFileRecoverablesForCurrentCheckpoint; /** * Constructor to create a new empty bucket. */ private Bucket( - final RecoverableWriter fsWriter, final int subtaskIndex, final BucketID bucketId, final Path bucketPath, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final OutputFileConfig outputFileConfig) { - this.fsWriter = checkNotNull(fsWriter); this.subtaskIndex = subtaskIndex; this.bucketId = checkNotNull(bucketId); this.bucketPath = checkNotNull(bucketPath); this.partCounter = initialPartCounter; - this.partFileFactory = checkNotNull(partFileFactory); + this.bucketWriter = checkNotNull(bucketWriter); this.rollingPolicy = checkNotNull(rollingPolicy); - this.pendingPartsForCurrentCheckpoint = new ArrayList<>(); - this.pendingPartsPerCheckpoint = new TreeMap<>(); - this.resumablesPerCheckpoint = new TreeMap<>(); + this.pendingFileRecoverablesForCurrentCheckpoint = new ArrayList<>(); + this.pendingFileRecoverablesPerCheckpoint = new TreeMap<>(); + this.inProgressFileRecoverablesPerCheckpoint = new TreeMap<>(); this.outputFileConfig = checkNotNull(outputFileConfig); } @@ -110,16 +102,14 @@ private Bucket( * Constructor to restore a bucket from checkpointed state. */ private Bucket( - final RecoverableWriter fsWriter, final int subtaskIndex, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileFactory, + final BucketWriter partFileFactory, final RollingPolicy rollingPolicy, final BucketState bucketState, final OutputFileConfig outputFileConfig) throws IOException { this( - fsWriter, subtaskIndex, bucketState.getBucketId(), bucketState.getBucketPath(), @@ -133,31 +123,29 @@ private Bucket( } private void restoreInProgressFile(final BucketState state) throws IOException { - if (!state.hasInProgressResumableFile()) { + if (!state.hasInProgressFileRecoverable()) { return; } // we try to resume the previous in-progress file - final ResumeRecoverable resumable = state.getInProgressResumableFile(); + final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable(); - if (fsWriter.supportsResume()) { - final RecoverableFsDataOutputStream stream = fsWriter.recover(resumable); - inProgressPart = partFileFactory.resumeFrom( - bucketId, stream, resumable, state.getInProgressFileCreationTime()); + if (bucketWriter.getProperties().supportsResume()) { + inProgressPart = bucketWriter.resumeInProgressFileFrom( + bucketId, inProgressFileRecoverable, state.getInProgressFileCreationTime()); } else { // if the writer does not support resume, then we close the // in-progress part and commit it, as done in the case of pending files. - - fsWriter.recoverForCommit(resumable).commitAfterRecovery(); + bucketWriter.recoverPendingFile(inProgressFileRecoverable).commitAfterRecovery(); } } private void commitRecoveredPendingFiles(final BucketState state) throws IOException { // we commit pending files for checkpoints that precess the last successful one, from which we are recovering - for (List committables: state.getCommittableFilesPerCheckpoint().values()) { - for (CommitRecoverable committable: committables) { - fsWriter.recoverForCommit(committable).commitAfterRecovery(); + for (List pendingFileRecoverables: state.getPendingFileRecoverablesPerCheckpoint().values()) { + for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable: pendingFileRecoverables) { + bucketWriter.recoverPendingFile(pendingFileRecoverable).commitAfterRecovery(); } } } @@ -175,7 +163,7 @@ public long getPartCounter() { } boolean isActive() { - return inProgressPart != null || !pendingPartsForCurrentCheckpoint.isEmpty() || !pendingPartsPerCheckpoint.isEmpty(); + return inProgressPart != null || !pendingFileRecoverablesForCurrentCheckpoint.isEmpty() || !pendingFileRecoverablesPerCheckpoint.isEmpty(); } void merge(final Bucket bucket) throws IOException { @@ -184,16 +172,16 @@ void merge(final Bucket bucket) throws IOException { // There should be no pending files in the "to-merge" states. // The reason is that: - // 1) the pendingPartsForCurrentCheckpoint is emptied whenever we take a snapshot (see prepareBucketForCheckpointing()). - // So a snapshot, including the one we are recovering from, will never contain such files. - // 2) the files in pendingPartsPerCheckpoint are committed upon recovery (see commitRecoveredPendingFiles()). + // 1) the pendingFileRecoverablesForCurrentCheckpoint is emptied whenever we take a Recoverable (see prepareBucketForCheckpointing()). + // So a Recoverable, including the one we are recovering from, will never contain such files. + // 2) the files in pendingFileRecoverablesPerCheckpoint are committed upon recovery (see commitRecoveredPendingFiles()). - checkState(bucket.pendingPartsForCurrentCheckpoint.isEmpty()); - checkState(bucket.pendingPartsPerCheckpoint.isEmpty()); + checkState(bucket.pendingFileRecoverablesForCurrentCheckpoint.isEmpty()); + checkState(bucket.pendingFileRecoverablesPerCheckpoint.isEmpty()); - CommitRecoverable committable = bucket.closePartFile(); - if (committable != null) { - pendingPartsForCurrentCheckpoint.add(committable); + InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable = bucket.closePartFile(); + if (pendingFileRecoverable != null) { + pendingFileRecoverablesForCurrentCheckpoint.add(pendingFileRecoverable); } if (LOG.isDebugEnabled()) { @@ -218,8 +206,7 @@ private void rollPartFile(final long currentTime) throws IOException { closePartFile(); final Path partFilePath = assembleNewPartPath(); - final RecoverableFsDataOutputStream stream = fsWriter.open(partFilePath); - inProgressPart = partFileFactory.openNew(bucketId, stream, partFilePath, currentTime); + inProgressPart = bucketWriter.openNewInProgressFile(bucketId, partFilePath, currentTime); if (LOG.isDebugEnabled()) { LOG.debug("Subtask {} opening new part file \"{}\" for bucket id={}.", @@ -233,14 +220,14 @@ private Path assembleNewPartPath() { return new Path(bucketPath, outputFileConfig.getPartPrefix() + '-' + subtaskIndex + '-' + partCounter + outputFileConfig.getPartSuffix()); } - private CommitRecoverable closePartFile() throws IOException { - CommitRecoverable committable = null; + private InProgressFileWriter.PendingFileRecoverable closePartFile() throws IOException { + InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable = null; if (inProgressPart != null) { - committable = inProgressPart.closeForCommit(); - pendingPartsForCurrentCheckpoint.add(committable); + pendingFileRecoverable = inProgressPart.closeForCommit(); + pendingFileRecoverablesForCurrentCheckpoint.add(pendingFileRecoverable); inProgressPart = null; } - return committable; + return pendingFileRecoverable; } void disposePartFile() { @@ -252,24 +239,16 @@ void disposePartFile() { BucketState onReceptionOfCheckpoint(long checkpointId) throws IOException { prepareBucketForCheckpointing(checkpointId); - ResumeRecoverable inProgressResumable = null; + InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = null; long inProgressFileCreationTime = Long.MAX_VALUE; if (inProgressPart != null) { - inProgressResumable = inProgressPart.persist(); + inProgressFileRecoverable = inProgressPart.persist(); inProgressFileCreationTime = inProgressPart.getCreationTime(); - - // the following is an optimization so that writers that do not - // require cleanup, they do not have to keep track of resumables - // and later iterate over the active buckets. - // (see onSuccessfulCompletionOfCheckpoint()) - - if (fsWriter.requiresCleanupOfRecoverableState()) { - this.resumablesPerCheckpoint.put(checkpointId, inProgressResumable); - } + this.inProgressFileRecoverablesPerCheckpoint.put(checkpointId, inProgressFileRecoverable); } - return new BucketState<>(bucketId, bucketPath, inProgressFileCreationTime, inProgressResumable, pendingPartsPerCheckpoint); + return new BucketState<>(bucketId, bucketPath, inProgressFileCreationTime, inProgressFileRecoverable, pendingFileRecoverablesPerCheckpoint); } private void prepareBucketForCheckpointing(long checkpointId) throws IOException { @@ -280,49 +259,46 @@ private void prepareBucketForCheckpointing(long checkpointId) throws IOException closePartFile(); } - if (!pendingPartsForCurrentCheckpoint.isEmpty()) { - pendingPartsPerCheckpoint.put(checkpointId, pendingPartsForCurrentCheckpoint); - pendingPartsForCurrentCheckpoint = new ArrayList<>(); + if (!pendingFileRecoverablesForCurrentCheckpoint.isEmpty()) { + pendingFileRecoverablesPerCheckpoint.put(checkpointId, pendingFileRecoverablesForCurrentCheckpoint); + pendingFileRecoverablesForCurrentCheckpoint = new ArrayList<>(); } } void onSuccessfulCompletionOfCheckpoint(long checkpointId) throws IOException { - checkNotNull(fsWriter); + checkNotNull(bucketWriter); - Iterator>> it = - pendingPartsPerCheckpoint.headMap(checkpointId, true) + Iterator>> it = + pendingFileRecoverablesPerCheckpoint.headMap(checkpointId, true) .entrySet().iterator(); while (it.hasNext()) { - Map.Entry> entry = it.next(); + Map.Entry> entry = it.next(); - for (CommitRecoverable committable : entry.getValue()) { - fsWriter.recoverForCommit(committable).commit(); + for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable : entry.getValue()) { + bucketWriter.recoverPendingFile(pendingFileRecoverable).commit(); } it.remove(); } - cleanupOutdatedResumables(checkpointId); + cleanupInProgressFileRecoverables(checkpointId); } - private void cleanupOutdatedResumables(long checkpointId) throws IOException { - Iterator> it = - resumablesPerCheckpoint.headMap(checkpointId, false) + private void cleanupInProgressFileRecoverables(long checkpointId) throws IOException { + Iterator> it = + inProgressFileRecoverablesPerCheckpoint.headMap(checkpointId, false) .entrySet().iterator(); while (it.hasNext()) { - final ResumeRecoverable recoverable = it.next().getValue(); + final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = it.next().getValue(); - // this check is redundant, as we only put entries in the resumablesPerCheckpoint map - // list when the requiresCleanupOfRecoverableState() returns true, but having it makes + // this check is redundant, as we only put entries in the inProgressFileRecoverablesPerCheckpoint map + // list when the requiresCleanupOfInProgressFileRecoverableState() returns true, but having it makes // the code more readable. - if (fsWriter.requiresCleanupOfRecoverableState()) { - final boolean successfullyDeleted = fsWriter.cleanupRecoverableState(recoverable); - - if (LOG.isDebugEnabled() && successfullyDeleted) { - LOG.debug("Subtask {} successfully deleted incomplete part for bucket id={}.", subtaskIndex, bucketId); - } + final boolean successfullyDeleted = bucketWriter.cleanupInProgressFileRecoverable(inProgressFileRecoverable); + if (LOG.isDebugEnabled() && successfullyDeleted) { + LOG.debug("Subtask {} successfully deleted incomplete part for bucket id={}.", subtaskIndex, bucketId); } it.remove(); } @@ -342,54 +318,51 @@ void onProcessingTime(long timestamp) throws IOException { // --------------------------- Testing Methods ----------------------------- @VisibleForTesting - Map> getPendingPartsPerCheckpoint() { - return pendingPartsPerCheckpoint; + Map> getPendingFileRecoverablesPerCheckpoint() { + return pendingFileRecoverablesPerCheckpoint; } @Nullable @VisibleForTesting - PartFileWriter getInProgressPart() { + InProgressFileWriter getInProgressPart() { return inProgressPart; } @VisibleForTesting - List getPendingPartsForCurrentCheckpoint() { - return pendingPartsForCurrentCheckpoint; + List getPendingFileRecoverablesForCurrentCheckpoint() { + return pendingFileRecoverablesForCurrentCheckpoint; } // --------------------------- Static Factory Methods ----------------------------- /** * Creates a new empty {@code Bucket}. - * @param fsWriter the filesystem-specific {@link RecoverableWriter}. * @param subtaskIndex the index of the subtask creating the bucket. * @param bucketId the identifier of the bucket, as returned by the {@link BucketAssigner}. * @param bucketPath the path to where the part files for the bucket will be written to. * @param initialPartCounter the initial counter for the part files of the bucket. - * @param partFileFactory the {@link PartFileWriter.PartFileFactory} the factory creating part file writers. + * @param bucketWriter the {@link BucketWriter} used to write part files in the bucket. * @param the type of input elements to the sink. * @param the type of the identifier of the bucket, as returned by the {@link BucketAssigner} * @param outputFileConfig the part file configuration. * @return The new Bucket. */ static Bucket getNew( - final RecoverableWriter fsWriter, final int subtaskIndex, final BucketID bucketId, final Path bucketPath, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final OutputFileConfig outputFileConfig) { - return new Bucket<>(fsWriter, subtaskIndex, bucketId, bucketPath, initialPartCounter, partFileFactory, rollingPolicy, outputFileConfig); + return new Bucket<>(subtaskIndex, bucketId, bucketPath, initialPartCounter, bucketWriter, rollingPolicy, outputFileConfig); } /** * Restores a {@code Bucket} from the state included in the provided {@link BucketState}. - * @param fsWriter the filesystem-specific {@link RecoverableWriter}. * @param subtaskIndex the index of the subtask creating the bucket. * @param initialPartCounter the initial counter for the part files of the bucket. - * @param partFileFactory the {@link PartFileWriter.PartFileFactory} the factory creating part file writers. + * @param bucketWriter the {@link BucketWriter} used to write part files in the bucket. * @param bucketState the initial state of the restored bucket. * @param the type of input elements to the sink. * @param the type of the identifier of the bucket, as returned by the {@link BucketAssigner} @@ -397,13 +370,12 @@ static Bucket getNew( * @return The restored Bucket. */ static Bucket restore( - final RecoverableWriter fsWriter, final int subtaskIndex, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final BucketState bucketState, final OutputFileConfig outputFileConfig) throws IOException { - return new Bucket<>(fsWriter, subtaskIndex, initialPartCounter, partFileFactory, rollingPolicy, bucketState, outputFileConfig); + return new Bucket<>(subtaskIndex, initialPartCounter, bucketWriter, rollingPolicy, bucketState, outputFileConfig); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketFactory.java index 260e82c796011..64236274f841c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketFactory.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketFactory.java @@ -20,7 +20,6 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableWriter; import java.io.IOException; import java.io.Serializable; @@ -32,20 +31,18 @@ interface BucketFactory extends Serializable { Bucket getNewBucket( - final RecoverableWriter fsWriter, final int subtaskIndex, final BucketID bucketId, final Path bucketPath, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileWriterFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final OutputFileConfig outputFileConfig) throws IOException; Bucket restoreBucket( - final RecoverableWriter fsWriter, final int subtaskIndex, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileWriterFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final BucketState bucketState, final OutputFileConfig outputFileConfig) throws IOException; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketState.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketState.java index 1829381506892..75c00b9834a7d 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketState.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketState.java @@ -46,30 +46,30 @@ class BucketState { private final long inProgressFileCreationTime; /** - * A {@link RecoverableWriter.ResumeRecoverable} for the currently open + * A {@link InProgressFileWriter.InProgressFileRecoverable} for the currently open * part file, or null if there is no currently open part file. */ @Nullable - private final RecoverableWriter.ResumeRecoverable inProgressResumableFile; + private final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable; /** * The {@link RecoverableWriter.CommitRecoverable files} pending to be * committed, organized by checkpoint id. */ - private final Map> committableFilesPerCheckpoint; + private final Map> pendingFileRecoverablesPerCheckpoint; BucketState( final BucketID bucketId, final Path bucketPath, final long inProgressFileCreationTime, - @Nullable final RecoverableWriter.ResumeRecoverable inProgressResumableFile, - final Map> pendingCommittablesPerCheckpoint + @Nullable final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable, + final Map> pendingFileRecoverablesPerCheckpoint ) { this.bucketId = Preconditions.checkNotNull(bucketId); this.bucketPath = Preconditions.checkNotNull(bucketPath); this.inProgressFileCreationTime = inProgressFileCreationTime; - this.inProgressResumableFile = inProgressResumableFile; - this.committableFilesPerCheckpoint = Preconditions.checkNotNull(pendingCommittablesPerCheckpoint); + this.inProgressFileRecoverable = inProgressFileRecoverable; + this.pendingFileRecoverablesPerCheckpoint = Preconditions.checkNotNull(pendingFileRecoverablesPerCheckpoint); } BucketID getBucketId() { @@ -84,17 +84,17 @@ long getInProgressFileCreationTime() { return inProgressFileCreationTime; } - boolean hasInProgressResumableFile() { - return inProgressResumableFile != null; + boolean hasInProgressFileRecoverable() { + return inProgressFileRecoverable != null; } @Nullable - RecoverableWriter.ResumeRecoverable getInProgressResumableFile() { - return inProgressResumableFile; + InProgressFileWriter.InProgressFileRecoverable getInProgressFileRecoverable() { + return inProgressFileRecoverable; } - Map> getCommittableFilesPerCheckpoint() { - return committableFilesPerCheckpoint; + Map> getPendingFileRecoverablesPerCheckpoint() { + return pendingFileRecoverablesPerCheckpoint; } @Override @@ -105,13 +105,13 @@ public String toString() { .append("BucketState for bucketId=").append(bucketId) .append(" and bucketPath=").append(bucketPath); - if (hasInProgressResumableFile()) { + if (hasInProgressFileRecoverable()) { strBuilder.append(", has open part file created @ ").append(inProgressFileCreationTime); } - if (!committableFilesPerCheckpoint.isEmpty()) { + if (!pendingFileRecoverablesPerCheckpoint.isEmpty()) { strBuilder.append(", has pending files for checkpoints: {"); - for (long checkpointId: committableFilesPerCheckpoint.keySet()) { + for (long checkpointId: pendingFileRecoverablesPerCheckpoint.keySet()) { strBuilder.append(checkpointId).append(' '); } strBuilder.append('}'); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializer.java index 04de2462d6709..5863a037d4bf5 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializer.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializer.java @@ -19,7 +19,6 @@ package org.apache.flink.streaming.api.functions.sink.filesystem; import org.apache.flink.annotation.Internal; -import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.fs.Path; import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerialization; @@ -45,119 +44,172 @@ class BucketStateSerializer implements SimpleVersionedSerializer resumableSerializer; + private final SimpleVersionedSerializer inProgressFileRecoverableSerializer; - private final SimpleVersionedSerializer commitableSerializer; + private final SimpleVersionedSerializer pendingFileRecoverableSerializer; private final SimpleVersionedSerializer bucketIdSerializer; BucketStateSerializer( - final SimpleVersionedSerializer resumableSerializer, - final SimpleVersionedSerializer commitableSerializer, + final SimpleVersionedSerializer inProgressFileRecoverableSerializer, + final SimpleVersionedSerializer pendingFileRecoverableSerializer, final SimpleVersionedSerializer bucketIdSerializer ) { - this.resumableSerializer = Preconditions.checkNotNull(resumableSerializer); - this.commitableSerializer = Preconditions.checkNotNull(commitableSerializer); + this.inProgressFileRecoverableSerializer = Preconditions.checkNotNull(inProgressFileRecoverableSerializer); + this.pendingFileRecoverableSerializer = Preconditions.checkNotNull(pendingFileRecoverableSerializer); this.bucketIdSerializer = Preconditions.checkNotNull(bucketIdSerializer); } @Override public int getVersion() { - return 1; + return 2; } @Override public byte[] serialize(BucketState state) throws IOException { DataOutputSerializer out = new DataOutputSerializer(256); out.writeInt(MAGIC_NUMBER); - serializeV1(state, out); + serializeV2(state, out); return out.getCopyOfBuffer(); } @Override public BucketState deserialize(int version, byte[] serialized) throws IOException { + final DataInputDeserializer in = new DataInputDeserializer(serialized); + switch (version) { case 1: - DataInputDeserializer in = new DataInputDeserializer(serialized); validateMagicNumber(in); return deserializeV1(in); + case 2: + validateMagicNumber(in); + return deserializeV2(in); default: throw new IOException("Unrecognized version or corrupt state: " + version); } } - @VisibleForTesting - void serializeV1(BucketState state, DataOutputView out) throws IOException { - SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); - out.writeUTF(state.getBucketPath().toString()); - out.writeLong(state.getInProgressFileCreationTime()); + private void serializeV2(BucketState state, DataOutputView dataOutputView) throws IOException { + SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), dataOutputView); + dataOutputView.writeUTF(state.getBucketPath().toString()); + dataOutputView.writeLong(state.getInProgressFileCreationTime()); // put the current open part file - if (state.hasInProgressResumableFile()) { - final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); - out.writeBoolean(true); - SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); - } - else { - out.writeBoolean(false); + if (state.hasInProgressFileRecoverable()) { + final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable(); + dataOutputView.writeBoolean(true); + SimpleVersionedSerialization.writeVersionAndSerialize(inProgressFileRecoverableSerializer, inProgressFileRecoverable, dataOutputView); + } else { + dataOutputView.writeBoolean(false); } // put the map of pending files per checkpoint - final Map> pendingCommitters = state.getCommittableFilesPerCheckpoint(); + final Map> pendingFileRecoverables = state.getPendingFileRecoverablesPerCheckpoint(); - // manually keep the version here to safe some bytes - out.writeInt(commitableSerializer.getVersion()); + dataOutputView.writeInt(pendingFileRecoverableSerializer.getVersion()); - out.writeInt(pendingCommitters.size()); - for (Entry> resumablesForCheckpoint : pendingCommitters.entrySet()) { - List resumables = resumablesForCheckpoint.getValue(); + dataOutputView.writeInt(pendingFileRecoverables.size()); - out.writeLong(resumablesForCheckpoint.getKey()); - out.writeInt(resumables.size()); + for (Entry> pendingFilesForCheckpoint : pendingFileRecoverables.entrySet()) { + final List pendingFileRecoverableList = pendingFilesForCheckpoint.getValue(); - for (RecoverableWriter.CommitRecoverable resumable : resumables) { - byte[] serialized = commitableSerializer.serialize(resumable); - out.writeInt(serialized.length); - out.write(serialized); + dataOutputView.writeLong(pendingFilesForCheckpoint.getKey()); + dataOutputView.writeInt(pendingFileRecoverableList.size()); + + for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable : pendingFileRecoverableList) { + byte[] serialized = pendingFileRecoverableSerializer.serialize(pendingFileRecoverable); + dataOutputView.writeInt(serialized.length); + dataOutputView.write(serialized); } } } - @VisibleForTesting - BucketState deserializeV1(DataInputView in) throws IOException { + private BucketState deserializeV1(DataInputView in) throws IOException { + + final SimpleVersionedSerializer commitableSerializer = getCommitableSerializer(); + final SimpleVersionedSerializer resumableSerializer = getResumableSerializer(); + final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in); final String bucketPathStr = in.readUTF(); final long creationTime = in.readLong(); // then get the current resumable stream - RecoverableWriter.ResumeRecoverable current = null; + InProgressFileWriter.InProgressFileRecoverable current = null; if (in.readBoolean()) { - current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in); + current = + new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable( + SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in)); } final int committableVersion = in.readInt(); final int numCheckpoints = in.readInt(); - final HashMap> resumablesPerCheckpoint = new HashMap<>(numCheckpoints); + final HashMap> pendingFileRecoverablePerCheckpoint = new HashMap<>(numCheckpoints); for (int i = 0; i < numCheckpoints; i++) { final long checkpointId = in.readLong(); final int noOfResumables = in.readInt(); - final List resumables = new ArrayList<>(noOfResumables); + final List pendingFileRecoverables = new ArrayList<>(noOfResumables); for (int j = 0; j < noOfResumables; j++) { final byte[] bytes = new byte[in.readInt()]; in.readFully(bytes); - resumables.add(commitableSerializer.deserialize(committableVersion, bytes)); + pendingFileRecoverables.add( + new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(commitableSerializer.deserialize(committableVersion, bytes))); } - resumablesPerCheckpoint.put(checkpointId, resumables); + pendingFileRecoverablePerCheckpoint.put(checkpointId, pendingFileRecoverables); } return new BucketState<>( - bucketId, - new Path(bucketPathStr), - creationTime, - current, - resumablesPerCheckpoint); + bucketId, + new Path(bucketPathStr), + creationTime, + current, + pendingFileRecoverablePerCheckpoint); + } + + private BucketState deserializeV2(DataInputView dataInputView) throws IOException { + final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, dataInputView); + final String bucketPathStr = dataInputView.readUTF(); + final long creationTime = dataInputView.readLong(); + + // then get the current resumable stream + InProgressFileWriter.InProgressFileRecoverable current = null; + if (dataInputView.readBoolean()) { + current = SimpleVersionedSerialization.readVersionAndDeSerialize(inProgressFileRecoverableSerializer, dataInputView); + } + + final int pendingFileRecoverableSerializerVersion = dataInputView.readInt(); + final int numCheckpoints = dataInputView.readInt(); + final HashMap> pendingFileRecoverablesPerCheckpoint = new HashMap<>(numCheckpoints); + + for (int i = 0; i < numCheckpoints; i++) { + final long checkpointId = dataInputView.readLong(); + final int numOfPendingFileRecoverables = dataInputView.readInt(); + + final List pendingFileRecoverables = new ArrayList<>(numOfPendingFileRecoverables); + for (int j = 0; j < numOfPendingFileRecoverables; j++) { + final byte[] bytes = new byte[dataInputView.readInt()]; + dataInputView.readFully(bytes); + pendingFileRecoverables.add(pendingFileRecoverableSerializer.deserialize(pendingFileRecoverableSerializerVersion, bytes)); + } + pendingFileRecoverablesPerCheckpoint.put(checkpointId, pendingFileRecoverables); + } + + return new BucketState<>(bucketId, new Path(bucketPathStr), creationTime, current, pendingFileRecoverablesPerCheckpoint); + } + + private SimpleVersionedSerializer getResumableSerializer() { + final OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverableSerializer + outputStreamBasedInProgressFileRecoverableSerializer = + (OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverableSerializer) inProgressFileRecoverableSerializer; + return outputStreamBasedInProgressFileRecoverableSerializer.getResumeSerializer(); + } + + private SimpleVersionedSerializer getCommitableSerializer() { + final OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverableSerializer + outputStreamBasedPendingFileRecoverableSerializer = + (OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverableSerializer) pendingFileRecoverableSerializer; + return outputStreamBasedPendingFileRecoverableSerializer.getCommitSerializer(); } private static void validateMagicNumber(DataInputView in) throws IOException { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketWriter.java new file mode 100644 index 0000000000000..ed3a0e292f231 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketWriter.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.fs.Path; + +import java.io.IOException; + +/** + * An interface for factories that create the different {@link InProgressFileWriter writers}. + */ +@Internal +interface BucketWriter { + + /** + * Used to create a new {@link InProgressFileWriter}. + * @param bucketID the id of the bucket this writer is writing to. + * @param path the path this writer will write to. + * @param creationTime the creation time of the file. + * @return the new {@link InProgressFileWriter} + * @throws IOException Thrown if creating a writer fails. + */ + InProgressFileWriter openNewInProgressFile( + final BucketID bucketID, + final Path path, + final long creationTime) throws IOException; + + /** + * Used to resume a {@link InProgressFileWriter} from a {@link InProgressFileWriter.InProgressFileRecoverable}. + * @param bucketID the id of the bucket this writer is writing to. + * @param inProgressFileSnapshot the state of the part file. + * @param creationTime the creation time of the file. + * @return the resumed {@link InProgressFileWriter} + * @throws IOException Thrown if resuming a writer fails. + */ + InProgressFileWriter resumeInProgressFileFrom( + final BucketID bucketID, + final InProgressFileWriter.InProgressFileRecoverable inProgressFileSnapshot, + final long creationTime) throws IOException; + + /** + * @return the property of the {@link BucketWriter} + */ + WriterProperties getProperties(); + + /** + * Recovers a pending file for finalizing and committing. + * @param pendingFileRecoverable The handle with the recovery information. + * @return A pending file + * @throws IOException Thrown if recovering a pending file fails. + */ + PendingFile recoverPendingFile(final InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable) throws IOException; + + /** + * Frees up any resources that were previously occupied in order to be able to + * recover from a (potential) failure. + * + *

NOTE: This operation should not throw an exception, but return false if the cleanup did not + * happen for any reason. + * + * @param inProgressFileRecoverable the {@link InProgressFileWriter.InProgressFileRecoverable} whose state we want to clean-up. + * @return {@code true} if the resources were successfully freed, {@code false} otherwise + * (e.g. the file to be deleted was not there for any reason - already deleted or never created). + * @throws IOException if an I/O error occurs + */ + boolean cleanupInProgressFileRecoverable(final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable) throws IOException; + + /** + * This represents the file that can not write any data to. + */ + interface PendingFile { + /** + * Commits the pending file, making it visible. The file will contain the exact data + * as when the pending file was created. + * + * @throws IOException Thrown if committing fails. + */ + void commit() throws IOException; + + /** + * Commits the pending file, making it visible. The file will contain the exact data + * as when the pending file was created. + * + *

This method tolerates situations where the file was already committed and + * will not raise an exception in that case. This is important for idempotent + * commit retries as they need to happen after recovery. + * + * @throws IOException Thrown if committing fails. + */ + void commitAfterRecovery() throws IOException; + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java index f055798925aff..0c9b73fb17163 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/Buckets.java @@ -21,9 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.state.ListState; -import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerialization; import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.util.Preconditions; @@ -61,7 +59,7 @@ public class Buckets { private final BucketAssigner bucketAssigner; - private final PartFileWriter.PartFileFactory partFileWriterFactory; + private final BucketWriter bucketWriter; private final RollingPolicy rollingPolicy; @@ -78,8 +76,6 @@ public class Buckets { private long maxPartCounter; - private final RecoverableWriter fsWriter; - private final OutputFileConfig outputFileConfig; // --------------------------- State Related Fields ----------------------------- @@ -92,23 +88,23 @@ public class Buckets { * @param basePath The base path for our buckets. * @param bucketAssigner The {@link BucketAssigner} provided by the user. * @param bucketFactory The {@link BucketFactory} to be used to create buckets. - * @param partFileWriterFactory The {@link PartFileWriter.PartFileFactory} to be used when writing data. + * @param bucketWriter The {@link BucketWriter} to be used when writing data. * @param rollingPolicy The {@link RollingPolicy} as specified by the user. */ Buckets( final Path basePath, final BucketAssigner bucketAssigner, final BucketFactory bucketFactory, - final PartFileWriter.PartFileFactory partFileWriterFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, @Nullable final BucketLifeCycleListener bucketLifeCycleListener, final int subtaskIndex, - final OutputFileConfig outputFileConfig) throws IOException { + final OutputFileConfig outputFileConfig) { this.basePath = Preconditions.checkNotNull(basePath); this.bucketAssigner = Preconditions.checkNotNull(bucketAssigner); this.bucketFactory = Preconditions.checkNotNull(bucketFactory); - this.partFileWriterFactory = Preconditions.checkNotNull(partFileWriterFactory); + this.bucketWriter = Preconditions.checkNotNull(bucketWriter); this.rollingPolicy = Preconditions.checkNotNull(rollingPolicy); this.bucketLifeCycleListener = bucketLifeCycleListener; this.subtaskIndex = subtaskIndex; @@ -118,19 +114,10 @@ public class Buckets { this.activeBuckets = new HashMap<>(); this.bucketerContext = new Buckets.BucketerContext(); - try { - this.fsWriter = FileSystem.get(basePath.toUri()).createRecoverableWriter(); - } catch (IOException e) { - LOG.error("Unable to create filesystem for path: {}", basePath); - throw e; - } - this.bucketStateSerializer = new BucketStateSerializer<>( - fsWriter.getResumeRecoverableSerializer(), - fsWriter.getCommitRecoverableSerializer(), - bucketAssigner.getSerializer() - ); - + bucketWriter.getProperties().getInProgressFileRecoverableSerializer(), + bucketWriter.getProperties().getPendingFileRecoverableSerializer(), + bucketAssigner.getSerializer()); this.maxPartCounter = 0L; } @@ -185,10 +172,9 @@ private void handleRestoredBucketState(final BucketState recoveredStat final Bucket restoredBucket = bucketFactory .restoreBucket( - fsWriter, subtaskIndex, maxPartCounter, - partFileWriterFactory, + bucketWriter, rollingPolicy, recoveredState, outputFileConfig @@ -238,7 +224,7 @@ public void snapshotState( final ListState partCounterStateContainer) throws Exception { Preconditions.checkState( - fsWriter != null && bucketStateSerializer != null, + bucketWriter != null && bucketStateSerializer != null, "sink has not been initialized"); LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).", @@ -308,12 +294,11 @@ private Bucket getOrCreateBucketForBucketId(final BucketID bucketI if (bucket == null) { final Path bucketPath = assembleBucketPath(bucketId); bucket = bucketFactory.getNewBucket( - fsWriter, subtaskIndex, bucketId, bucketPath, maxPartCounter, - partFileWriterFactory, + bucketWriter, rollingPolicy, outputFileConfig); activeBuckets.put(bucketId, bucket); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkBucketWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkBucketWriter.java new file mode 100644 index 0000000000000..0f3cb9cac4046 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkBucketWriter.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.BulkWriter; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; + +/** + * A factory that creates {@link BulkPartWriter BulkPartWriters}. + * @param The type of input elements. + * @param The type of ids for the buckets, as returned by the {@link BucketAssigner}. + */ +@Internal +class BulkBucketWriter extends OutputStreamBasedPartFileWriter.OutputStreamBasedBucketWriter { + + private final BulkWriter.Factory writerFactory; + + BulkBucketWriter(final RecoverableWriter recoverableWriter, BulkWriter.Factory writerFactory) throws IOException { + super(recoverableWriter); + this.writerFactory = writerFactory; + } + + @Override + public InProgressFileWriter resumeFrom( + final BucketID bucketId, + final RecoverableFsDataOutputStream stream, + final RecoverableWriter.ResumeRecoverable resumable, + final long creationTime) throws IOException { + + Preconditions.checkNotNull(stream); + Preconditions.checkNotNull(resumable); + + final BulkWriter writer = writerFactory.create(stream); + return new BulkPartWriter<>(bucketId, stream, writer, creationTime); + } + + @Override + public InProgressFileWriter openNew( + final BucketID bucketId, + final RecoverableFsDataOutputStream stream, + final Path path, + final long creationTime) throws IOException { + + Preconditions.checkNotNull(stream); + Preconditions.checkNotNull(path); + + final BulkWriter writer = writerFactory.create(stream); + return new BulkPartWriter<>(bucketId, stream, writer, creationTime); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java index a44b0e8aea3da..b1b7864835eb1 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkPartWriter.java @@ -20,23 +20,21 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.serialization.BulkWriter; -import org.apache.flink.core.fs.Path; import org.apache.flink.core.fs.RecoverableFsDataOutputStream; -import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.util.Preconditions; import java.io.IOException; /** - * A {@link PartFileWriter} for bulk-encoding formats that use an {@link BulkPartWriter}. + * A {@link InProgressFileWriter} for bulk-encoding formats that use an {@link BulkPartWriter}. * This also implements the {@link PartFileInfo}. */ @Internal -final class BulkPartWriter extends PartFileWriter { +final class BulkPartWriter extends OutputStreamBasedPartFileWriter { private final BulkWriter writer; - private BulkPartWriter( + BulkPartWriter( final BucketID bucketId, final RecoverableFsDataOutputStream currentPartStream, final BulkWriter writer, @@ -46,62 +44,20 @@ private BulkPartWriter( } @Override - void write(IN element, long currentTime) throws IOException { + public void write(IN element, long currentTime) throws IOException { writer.addElement(element); markWrite(currentTime); } @Override - RecoverableWriter.ResumeRecoverable persist() { + public InProgressFileRecoverable persist() { throw new UnsupportedOperationException("Bulk Part Writers do not support \"pause and resume\" operations."); } @Override - RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { + public PendingFileRecoverable closeForCommit() throws IOException { writer.flush(); writer.finish(); return super.closeForCommit(); } - - /** - * A factory that creates {@link BulkPartWriter BulkPartWriters}. - * @param The type of input elements. - * @param The type of ids for the buckets, as returned by the {@link BucketAssigner}. - */ - static class Factory implements PartFileWriter.PartFileFactory { - - private final BulkWriter.Factory writerFactory; - - Factory(BulkWriter.Factory writerFactory) { - this.writerFactory = writerFactory; - } - - @Override - public PartFileWriter resumeFrom( - final BucketID bucketId, - final RecoverableFsDataOutputStream stream, - final RecoverableWriter.ResumeRecoverable resumable, - final long creationTime) throws IOException { - - Preconditions.checkNotNull(stream); - Preconditions.checkNotNull(resumable); - - final BulkWriter writer = writerFactory.create(stream); - return new BulkPartWriter<>(bucketId, stream, writer, creationTime); - } - - @Override - public PartFileWriter openNew( - final BucketID bucketId, - final RecoverableFsDataOutputStream stream, - final Path path, - final long creationTime) throws IOException { - - Preconditions.checkNotNull(stream); - Preconditions.checkNotNull(path); - - final BulkWriter writer = writerFactory.create(stream); - return new BulkPartWriter<>(bucketId, stream, writer, creationTime); - } - } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/DefaultBucketFactoryImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/DefaultBucketFactoryImpl.java index 529b93afa8c5e..bb20b975f0d4a 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/DefaultBucketFactoryImpl.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/DefaultBucketFactoryImpl.java @@ -20,7 +20,6 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableWriter; import java.io.IOException; @@ -34,41 +33,37 @@ class DefaultBucketFactoryImpl implements BucketFactory getNewBucket( - final RecoverableWriter fsWriter, final int subtaskIndex, final BucketID bucketId, final Path bucketPath, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileWriterFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final OutputFileConfig outputFileConfig) { return Bucket.getNew( - fsWriter, subtaskIndex, bucketId, bucketPath, initialPartCounter, - partFileWriterFactory, + bucketWriter, rollingPolicy, outputFileConfig); } @Override public Bucket restoreBucket( - final RecoverableWriter fsWriter, final int subtaskIndex, final long initialPartCounter, - final PartFileWriter.PartFileFactory partFileWriterFactory, + final BucketWriter bucketWriter, final RollingPolicy rollingPolicy, final BucketState bucketState, final OutputFileConfig outputFileConfig) throws IOException { return Bucket.restore( - fsWriter, subtaskIndex, initialPartCounter, - partFileWriterFactory, + bucketWriter, rollingPolicy, bucketState, outputFileConfig); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/InProgressFileWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/InProgressFileWriter.java new file mode 100644 index 0000000000000..60798d1809d31 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/InProgressFileWriter.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.annotation.Internal; + +import java.io.IOException; + +/** + * The {@link Bucket} uses the {@link InProgressFileWriter} to write element to a part file. + */ +@Internal +interface InProgressFileWriter extends PartFileInfo { + + /** + * Write a element to the part file. + * @param element the element to be written. + * @param currentTime the writing time. + * @throws IOException Thrown if writing the element fails. + */ + void write(final IN element, final long currentTime) throws IOException; + + /** + * @return The state of the current part file. + * @throws IOException Thrown if persisting the part file fails. + */ + InProgressFileRecoverable persist() throws IOException; + + + /** + * @return The state of the pending part file. {@link Bucket} uses this to commit the pending file. + * @throws IOException Thrown if an I/O error occurs. + */ + PendingFileRecoverable closeForCommit() throws IOException; + + /** + * Dispose the part file. + */ + void dispose(); + + // ------------------------------------------------------------------------ + + + /** + * A handle can be used to recover in-progress file.. + */ + interface InProgressFileRecoverable extends PendingFileRecoverable {} + + + /** + * The handle can be used to recover pending file. + */ + interface PendingFileRecoverable {} +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputStreamBasedPartFileWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputStreamBasedPartFileWriter.java new file mode 100644 index 0000000000000..2d8c4231f2927 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/OutputStreamBasedPartFileWriter.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; +import org.apache.flink.core.io.SimpleVersionedSerialization; +import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.IOUtils; + +import java.io.IOException; + +/** + * The base class for all the part file writer that use {@link org.apache.flink.core.fs.RecoverableFsDataOutputStream}. + * @param the element type + * @param the bucket type + */ +public abstract class OutputStreamBasedPartFileWriter extends AbstractPartFileWriter { + + final RecoverableFsDataOutputStream currentPartStream; + + OutputStreamBasedPartFileWriter( + final BucketID bucketID, + final RecoverableFsDataOutputStream recoverableFsDataOutputStream, + final long createTime) { + super(bucketID, createTime); + this.currentPartStream = recoverableFsDataOutputStream; + } + + @Override + public InProgressFileRecoverable persist() throws IOException { + return new OutputStreamBasedInProgressFileRecoverable(currentPartStream.persist()); + } + + @Override + public PendingFileRecoverable closeForCommit() throws IOException { + return new OutputStreamBasedPendingFileRecoverable(currentPartStream.closeForCommit().getRecoverable()); + } + + @Override + public void dispose() { + // we can suppress exceptions here, because we do not rely on close() to + // flush or persist any data + IOUtils.closeQuietly(currentPartStream); + } + + @Override + public long getSize() throws IOException { + return currentPartStream.getPos(); + } + + abstract static class OutputStreamBasedBucketWriter implements BucketWriter { + + private final RecoverableWriter recoverableWriter; + + OutputStreamBasedBucketWriter(final RecoverableWriter recoverableWriter) { + this.recoverableWriter = recoverableWriter; + } + + @Override + public InProgressFileWriter openNewInProgressFile(final BucketID bucketID, final Path path, final long creationTime) throws IOException { + return openNew(bucketID, recoverableWriter.open(path), path, creationTime); + } + + @Override + public InProgressFileWriter resumeInProgressFileFrom(final BucketID bucketID, final InProgressFileRecoverable inProgressFileRecoverable, final long creationTime) throws IOException { + final OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable = (OutputStreamBasedInProgressFileRecoverable) inProgressFileRecoverable; + return resumeFrom( + bucketID, + recoverableWriter.recover(outputStreamBasedInProgressRecoverable.getResumeRecoverable()), + outputStreamBasedInProgressRecoverable.getResumeRecoverable(), + creationTime); + } + + @Override + public PendingFile recoverPendingFile(final PendingFileRecoverable pendingFileRecoverable) throws IOException { + final RecoverableWriter.CommitRecoverable commitRecoverable; + + if (pendingFileRecoverable instanceof OutputStreamBasedPendingFileRecoverable) { + commitRecoverable = ((OutputStreamBasedPendingFileRecoverable) pendingFileRecoverable).getCommitRecoverable(); + } else if (pendingFileRecoverable instanceof OutputStreamBasedInProgressFileRecoverable) { + commitRecoverable = ((OutputStreamBasedInProgressFileRecoverable) pendingFileRecoverable).getResumeRecoverable(); + } else { + throw new IllegalArgumentException("can not recover from the pendingFileRecoverable"); + } + return new OutputStreamBasedPendingFile(recoverableWriter.recoverForCommit(commitRecoverable)); + } + + @Override + public boolean cleanupInProgressFileRecoverable(InProgressFileRecoverable inProgressFileRecoverable) throws IOException { + final RecoverableWriter.ResumeRecoverable resumeRecoverable = + ((OutputStreamBasedInProgressFileRecoverable) inProgressFileRecoverable).getResumeRecoverable(); + return recoverableWriter.cleanupRecoverableState(resumeRecoverable); + } + + @Override + public WriterProperties getProperties() { + return new WriterProperties( + new OutputStreamBasedInProgressFileRecoverableSerializer(recoverableWriter.getResumeRecoverableSerializer()), + new OutputStreamBasedPendingFileRecoverableSerializer(recoverableWriter.getCommitRecoverableSerializer()), + recoverableWriter.supportsResume()); + } + + public abstract InProgressFileWriter openNew( + final BucketID bucketId, + final RecoverableFsDataOutputStream stream, + final Path path, + final long creationTime) throws IOException; + + public abstract InProgressFileWriter resumeFrom( + final BucketID bucketId, + final RecoverableFsDataOutputStream stream, + final RecoverableWriter.ResumeRecoverable resumable, + final long creationTime) throws IOException; + } + + static final class OutputStreamBasedPendingFileRecoverable implements PendingFileRecoverable { + + private final RecoverableWriter.CommitRecoverable commitRecoverable; + + OutputStreamBasedPendingFileRecoverable(final RecoverableWriter.CommitRecoverable commitRecoverable) { + this.commitRecoverable = commitRecoverable; + } + + RecoverableWriter.CommitRecoverable getCommitRecoverable() { + return commitRecoverable; + } + } + + static final class OutputStreamBasedInProgressFileRecoverable implements InProgressFileRecoverable { + + private final RecoverableWriter.ResumeRecoverable resumeRecoverable; + + OutputStreamBasedInProgressFileRecoverable(final RecoverableWriter.ResumeRecoverable resumeRecoverable) { + this.resumeRecoverable = resumeRecoverable; + } + + RecoverableWriter.ResumeRecoverable getResumeRecoverable() { + return resumeRecoverable; + } + } + + static final class OutputStreamBasedPendingFile implements BucketWriter.PendingFile { + + private final RecoverableFsDataOutputStream.Committer committer; + + OutputStreamBasedPendingFile(final RecoverableFsDataOutputStream.Committer committer) { + this.committer = committer; + } + + @Override + public void commit() throws IOException { + committer.commit(); + } + + @Override + public void commitAfterRecovery() throws IOException { + committer.commitAfterRecovery(); + } + } + + static class OutputStreamBasedInProgressFileRecoverableSerializer implements SimpleVersionedSerializer { + + private static final int MAGIC_NUMBER = 0xb3a4073d; + + private final SimpleVersionedSerializer resumeSerializer; + + OutputStreamBasedInProgressFileRecoverableSerializer(SimpleVersionedSerializer resumeSerializer) { + this.resumeSerializer = resumeSerializer; + } + + @Override + public int getVersion() { + return 1; + } + + @Override + public byte[] serialize(InProgressFileRecoverable inProgressRecoverable) throws IOException { + OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable = (OutputStreamBasedInProgressFileRecoverable) inProgressRecoverable; + DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(256); + dataOutputSerializer.writeInt(MAGIC_NUMBER); + serializeV1(outputStreamBasedInProgressRecoverable, dataOutputSerializer); + return dataOutputSerializer.getCopyOfBuffer(); + } + + @Override + public InProgressFileRecoverable deserialize(int version, byte[] serialized) throws IOException { + switch (version) { + case 1: + DataInputView dataInputView = new DataInputDeserializer(serialized); + validateMagicNumber(dataInputView); + return deserializeV1(dataInputView); + default: + throw new IOException("Unrecognized version or corrupt state: " + version); + } + } + + SimpleVersionedSerializer getResumeSerializer() { + return resumeSerializer; + } + + private void serializeV1(final OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable, final DataOutputView dataOutputView) throws IOException { + SimpleVersionedSerialization.writeVersionAndSerialize(resumeSerializer, outputStreamBasedInProgressRecoverable.getResumeRecoverable(), dataOutputView); + } + + private OutputStreamBasedInProgressFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException { + return new OutputStreamBasedInProgressFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(resumeSerializer, dataInputView)); + } + + private static void validateMagicNumber(final DataInputView dataInputView) throws IOException { + final int magicNumber = dataInputView.readInt(); + if (magicNumber != MAGIC_NUMBER) { + throw new IOException(String.format("Corrupt data: Unexpected magic number %08X", magicNumber)); + } + } + } + + static class OutputStreamBasedPendingFileRecoverableSerializer implements SimpleVersionedSerializer { + + private static final int MAGIC_NUMBER = 0x2c853c89; + + private final SimpleVersionedSerializer commitSerializer; + + OutputStreamBasedPendingFileRecoverableSerializer(final SimpleVersionedSerializer commitSerializer) { + this.commitSerializer = commitSerializer; + } + + @Override + public int getVersion() { + return 1; + } + + @Override + public byte[] serialize(PendingFileRecoverable pendingFileRecoverable) throws IOException { + OutputStreamBasedPendingFileRecoverable outputStreamBasedPendingFileRecoverable = (OutputStreamBasedPendingFileRecoverable) pendingFileRecoverable; + DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(256); + dataOutputSerializer.writeInt(MAGIC_NUMBER); + serializeV1(outputStreamBasedPendingFileRecoverable, dataOutputSerializer); + return dataOutputSerializer.getCopyOfBuffer(); + } + + @Override + public PendingFileRecoverable deserialize(int version, byte[] serialized) throws IOException { + switch (version) { + case 1: + DataInputDeserializer in = new DataInputDeserializer(serialized); + validateMagicNumber(in); + return deserializeV1(in); + + default: + throw new IOException("Unrecognized version or corrupt state: " + version); + } + } + + SimpleVersionedSerializer getCommitSerializer() { + return this.commitSerializer; + } + + private void serializeV1(final OutputStreamBasedPendingFileRecoverable outputStreamBasedPendingFileRecoverable, final DataOutputView dataOutputView) throws IOException { + SimpleVersionedSerialization.writeVersionAndSerialize(commitSerializer, outputStreamBasedPendingFileRecoverable.getCommitRecoverable(), dataOutputView); + } + + private OutputStreamBasedPendingFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException { + return new OutputStreamBasedPendingFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(commitSerializer, dataInputView)); + } + + private static void validateMagicNumber(final DataInputView dataInputView) throws IOException { + final int magicNumber = dataInputView.readInt(); + if (magicNumber != MAGIC_NUMBER) { + throw new IOException(String.format("Corrupt data: Unexpected magic number %08X", magicNumber)); + } + } + } + +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java deleted file mode 100644 index 95a2978a4c601..0000000000000 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/PartFileWriter.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.api.functions.sink.filesystem; - -import org.apache.flink.annotation.Internal; -import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableFsDataOutputStream; -import org.apache.flink.core.fs.RecoverableWriter; -import org.apache.flink.util.IOUtils; -import org.apache.flink.util.Preconditions; - -import java.io.IOException; - -/** - * An abstract writer for the currently open part file in a specific {@link Bucket}. - * - *

Currently, there are two subclasses, of this class: - *

    - *
  1. One for row-wise formats: the {@link RowWisePartWriter}.
    2. - *
  2. One for bulk encoding formats: the {@link BulkPartWriter}.
    3. - *
- * - *

This also implements the {@link PartFileInfo}. - */ -@Internal -abstract class PartFileWriter implements PartFileInfo { - - private final BucketID bucketId; - - private final long creationTime; - - protected final RecoverableFsDataOutputStream currentPartStream; - - private long lastUpdateTime; - - protected PartFileWriter( - final BucketID bucketId, - final RecoverableFsDataOutputStream currentPartStream, - final long creationTime) { - - Preconditions.checkArgument(creationTime >= 0L); - this.bucketId = Preconditions.checkNotNull(bucketId); - this.currentPartStream = Preconditions.checkNotNull(currentPartStream); - this.creationTime = creationTime; - this.lastUpdateTime = creationTime; - } - - abstract void write(IN element, long currentTime) throws IOException; - - RecoverableWriter.ResumeRecoverable persist() throws IOException { - return currentPartStream.persist(); - } - - RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { - return currentPartStream.closeForCommit().getRecoverable(); - } - - void dispose() { - // we can suppress exceptions here, because we do not rely on close() to - // flush or persist any data - IOUtils.closeQuietly(currentPartStream); - } - - void markWrite(long now) { - this.lastUpdateTime = now; - } - - @Override - public BucketID getBucketId() { - return bucketId; - } - - @Override - public long getCreationTime() { - return creationTime; - } - - @Override - public long getSize() throws IOException { - return currentPartStream.getPos(); - } - - @Override - public long getLastUpdateTime() { - return lastUpdateTime; - } - - // ------------------------------------------------------------------------ - - /** - * An interface for factories that create the different {@link PartFileWriter writers}. - */ - interface PartFileFactory { - - /** - * Used upon recovery from a failure to recover a {@link PartFileWriter writer}. - * @param bucketId the id of the bucket this writer is writing to. - * @param stream the filesystem-specific output stream to use when writing to the filesystem. - * @param resumable the state of the stream we are resurrecting. - * @param creationTime the creation time of the stream. - * @return the recovered {@link PartFileWriter writer}. - * @throws IOException - */ - PartFileWriter resumeFrom( - final BucketID bucketId, - final RecoverableFsDataOutputStream stream, - final RecoverableWriter.ResumeRecoverable resumable, - final long creationTime) throws IOException; - - /** - * Used to create a new {@link PartFileWriter writer}. - * @param bucketId the id of the bucket this writer is writing to. - * @param stream the filesystem-specific output stream to use when writing to the filesystem. - * @param path the part this writer will write to. - * @param creationTime the creation time of the stream. - * @return the new {@link PartFileWriter writer}. - * @throws IOException - */ - PartFileWriter openNew( - final BucketID bucketId, - final RecoverableFsDataOutputStream stream, - final Path path, - final long creationTime) throws IOException; - } -} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWiseBucketWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWiseBucketWriter.java new file mode 100644 index 0000000000000..784f8be662d89 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWiseBucketWriter.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.Encoder; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.fs.RecoverableFsDataOutputStream; +import org.apache.flink.core.fs.RecoverableWriter; +import org.apache.flink.util.Preconditions; + +/** + * A factory that creates {@link RowWisePartWriter RowWisePartWriters}. + * @param The type of input elements. + * @param The type of ids for the buckets, as returned by the {@link BucketAssigner}. + */ +@Internal +class RowWiseBucketWriter extends OutputStreamBasedPartFileWriter.OutputStreamBasedBucketWriter { + + private final Encoder encoder; + + RowWiseBucketWriter(final RecoverableWriter recoverableWriter, final Encoder encoder) { + super(recoverableWriter); + this.encoder = encoder; + } + + @Override + public InProgressFileWriter resumeFrom( + final BucketID bucketId, + final RecoverableFsDataOutputStream stream, + final RecoverableWriter.ResumeRecoverable resumable, + final long creationTime) { + + Preconditions.checkNotNull(stream); + Preconditions.checkNotNull(resumable); + + return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime); + } + + @Override + public InProgressFileWriter openNew( + final BucketID bucketId, + final RecoverableFsDataOutputStream stream, + final Path path, + final long creationTime) { + + Preconditions.checkNotNull(stream); + Preconditions.checkNotNull(path); + + return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime); + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java index 05c160c262964..bed9ec769c4d7 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/RowWisePartWriter.java @@ -20,23 +20,21 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.serialization.Encoder; -import org.apache.flink.core.fs.Path; import org.apache.flink.core.fs.RecoverableFsDataOutputStream; -import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.util.Preconditions; import java.io.IOException; /** - * A {@link PartFileWriter} for row-wise formats that use an {@link Encoder}. + * A {@link InProgressFileWriter} for row-wise formats that use an {@link Encoder}. * This also implements the {@link PartFileInfo}. */ @Internal -final class RowWisePartWriter extends PartFileWriter { +final class RowWisePartWriter extends OutputStreamBasedPartFileWriter { private final Encoder encoder; - private RowWisePartWriter( + RowWisePartWriter( final BucketID bucketId, final RecoverableFsDataOutputStream currentPartStream, final Encoder encoder, @@ -46,48 +44,8 @@ private RowWisePartWriter( } @Override - void write(IN element, long currentTime) throws IOException { + public void write(final IN element, final long currentTime) throws IOException { encoder.encode(element, currentPartStream); markWrite(currentTime); } - - /** - * A factory that creates {@link RowWisePartWriter RowWisePartWriters}. - * @param The type of input elements. - * @param The type of ids for the buckets, as returned by the {@link BucketAssigner}. - */ - static class Factory implements PartFileWriter.PartFileFactory { - - private final Encoder encoder; - - Factory(Encoder encoder) { - this.encoder = encoder; - } - - @Override - public PartFileWriter resumeFrom( - final BucketID bucketId, - final RecoverableFsDataOutputStream stream, - final RecoverableWriter.ResumeRecoverable resumable, - final long creationTime) throws IOException { - - Preconditions.checkNotNull(stream); - Preconditions.checkNotNull(resumable); - - return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime); - } - - @Override - public PartFileWriter openNew( - final BucketID bucketId, - final RecoverableFsDataOutputStream stream, - final Path path, - final long creationTime) throws IOException { - - Preconditions.checkNotNull(stream); - Preconditions.checkNotNull(path); - - return new RowWisePartWriter<>(bucketId, stream, encoder, creationTime); - } - } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java index cb58529afc56d..64e441838598f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java @@ -277,7 +277,7 @@ public Buckets createBuckets(int subtaskIndex) throws IOException basePath, bucketAssigner, bucketFactory, - new RowWisePartWriter.Factory<>(encoder), + new RowWiseBucketWriter<>(FileSystem.get(basePath.toUri()).createRecoverableWriter(), encoder), rollingPolicy, bucketLifeCycleListener, subtaskIndex, @@ -397,7 +397,7 @@ public Buckets createBuckets(int subtaskIndex) throws IOException basePath, bucketAssigner, bucketFactory, - new BulkPartWriter.Factory<>(writerFactory), + new BulkBucketWriter<>(FileSystem.get(basePath.toUri()).createRecoverableWriter(), writerFactory), rollingPolicy, bucketLifeCycleListener, subtaskIndex, diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/WriterProperties.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/WriterProperties.java new file mode 100644 index 0000000000000..4fee03c5d2f94 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/WriterProperties.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * This class describes the property of the {@link BucketWriter}. + */ +@Internal +public class WriterProperties { + + private final SimpleVersionedSerializer inProgressFileRecoverableSerializer; + + private final SimpleVersionedSerializer pendingFileRecoverableSerializer; + + private final boolean supportsResume; + + WriterProperties( + SimpleVersionedSerializer inProgressFileRecoverableSerializer, + SimpleVersionedSerializer pendingFileRecoverableSerializer, + boolean supportsResume) { + this.inProgressFileRecoverableSerializer = checkNotNull(inProgressFileRecoverableSerializer); + this.pendingFileRecoverableSerializer = checkNotNull(pendingFileRecoverableSerializer); + this.supportsResume = supportsResume; + } + + /** + * @return Whether the {@link BucketWriter} support appending data to the restored the in-progress file or not. + */ + boolean supportsResume() { + return supportsResume; + } + + /** + * @return the serializer for the {@link InProgressFileWriter.PendingFileRecoverable}. + */ + SimpleVersionedSerializer getPendingFileRecoverableSerializer() { + return pendingFileRecoverableSerializer; + } + + /** + * @return the serializer for the {@link InProgressFileWriter.InProgressFileRecoverable}. + */ + SimpleVersionedSerializer getInProgressFileRecoverableSerializer() { + return inProgressFileRecoverableSerializer; + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java index f48e4676ca974..ff2cc5a83a3c9 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketAssignerITCases.java @@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.functions.sink.filesystem; import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.BasePathBucketAssigner; import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; @@ -54,7 +55,7 @@ public void testAssembleBucketPath() throws Exception { basePath, new BasePathBucketAssigner<>(), new DefaultBucketFactoryImpl<>(), - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + new RowWiseBucketWriter<>(FileSystem.get(basePath.toUri()).createRecoverableWriter(), new SimpleStringEncoder<>()), rollingPolicy, null, 0, diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java index 81c57663c89c0..cbd18c486dc38 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java @@ -21,7 +21,6 @@ import org.apache.flink.api.common.serialization.SimpleStringEncoder; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerialization; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer; @@ -136,7 +135,7 @@ public void testSerializationEmpty() throws IOException { Assert.assertEquals(testBucketPath, bucket.getBucketPath()); Assert.assertNull(bucket.getInProgressPart()); - Assert.assertTrue(bucket.getPendingPartsPerCheckpoint().isEmpty()); + Assert.assertTrue(bucket.getPendingFileRecoverablesPerCheckpoint().isEmpty()); } @Test @@ -266,8 +265,8 @@ private void testDeserializationFull(final boolean withInProgress, final String final int noOfPendingCheckpoints = 5; // there are 5 checkpoint does not complete. - final Map> - pendingFileRecoverables = recoveredState.getCommittableFilesPerCheckpoint(); + final Map> + pendingFileRecoverables = recoveredState.getPendingFileRecoverablesPerCheckpoint(); Assert.assertEquals(5L, pendingFileRecoverables.size()); final Set beforeRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)) @@ -283,7 +282,7 @@ private void testDeserializationFull(final boolean withInProgress, final String // recover and commit final Bucket bucket = restoreBucket(noOfPendingCheckpoints + 1, recoveredState); Assert.assertEquals(testBucketPath, bucket.getBucketPath()); - Assert.assertEquals(0, bucket.getPendingPartsPerCheckpoint().size()); + Assert.assertEquals(0, bucket.getPendingFileRecoverablesForCurrentCheckpoint().size()); final Set afterRestorePaths = Files.list(outputPath.resolve(BUCKET_ID)) .map(file -> file.getFileName().toString()) @@ -313,27 +312,37 @@ private void testDeserializationFull(final boolean withInProgress, final String private static Bucket createNewBucket(final Path bucketPath) throws IOException { return Bucket.getNew( - FileSystem.getLocalFileSystem().createRecoverableWriter(), 0, BUCKET_ID, bucketPath, 0, - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + createBucketWriter(), DefaultRollingPolicy.builder().withMaxPartSize(10).build(), OutputFileConfig.builder().build()); } private static Bucket restoreBucket(final int initialPartCounter, final BucketState bucketState) throws IOException { return Bucket.restore( - FileSystem.getLocalFileSystem().createRecoverableWriter(), 0, initialPartCounter, - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + createBucketWriter(), DefaultRollingPolicy.builder().withMaxPartSize(10).build(), bucketState, OutputFileConfig.builder().build()); } + private static RowWiseBucketWriter createBucketWriter() throws IOException { + return new RowWiseBucketWriter<>(FileSystem.getLocalFileSystem().createRecoverableWriter(), new SimpleStringEncoder<>()); + } + + private static SimpleVersionedSerializer> bucketStateSerializer() throws IOException { + final RowWiseBucketWriter bucketWriter = createBucketWriter(); + return new BucketStateSerializer<>( + bucketWriter.getProperties().getInProgressFileRecoverableSerializer(), + bucketWriter.getProperties().getPendingFileRecoverableSerializer(), + SimpleVersionedStringSerializer.INSTANCE); + } + private static BucketState readBucketState(final String scenarioName, final int version) throws IOException { byte[] bytes = Files.readAllBytes(getSnapshotPath(scenarioName, version)); return SimpleVersionedSerialization.readVersionAndDeSerialize(bucketStateSerializer(), bytes); @@ -351,14 +360,6 @@ private static BucketState readBucketStateFromTemplate(final String scen return readBucketState(scenarioName, version); } - private static SimpleVersionedSerializer> bucketStateSerializer() throws IOException { - RecoverableWriter recoverableWriter = FileSystem.getLocalFileSystem().createRecoverableWriter(); - return new BucketStateSerializer<>( - recoverableWriter.getResumeRecoverableSerializer(), - recoverableWriter.getCommitRecoverableSerializer(), - SimpleVersionedStringSerializer.INSTANCE); - } - private static void moveToTemplateDirectory(java.nio.file.Path scenarioPath) throws IOException { FileUtils.copy(new Path(scenarioPath.toString()), new Path(scenarioPath.toString() + "-template"), false); FileUtils.deleteDirectory(scenarioPath.toFile()); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java index ee85e556f5358..a4d9a09e43e62 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketTest.java @@ -18,6 +18,7 @@ package org.apache.flink.streaming.api.functions.sink.filesystem; +import org.apache.flink.api.common.serialization.Encoder; import org.apache.flink.api.common.serialization.SimpleStringEncoder; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; @@ -171,7 +172,7 @@ private static TypeSafeMatcher> hasActiveInProgressFile() { return new TypeSafeMatcher>() { @Override protected boolean matchesSafely(BucketState state) { - return state.getInProgressResumableFile() != null; + return state.getInProgressFileRecoverable() != null; } @Override @@ -185,7 +186,7 @@ private static TypeSafeMatcher> hasNoActiveInProgressFile() return new TypeSafeMatcher>() { @Override protected boolean matchesSafely(BucketState state) { - return state.getInProgressResumableFile() == null; + return state.getInProgressFileRecoverable() == null; } @Override @@ -200,7 +201,7 @@ private static TypeSafeMatcher> hasNullInProgressFile(fin return new TypeSafeMatcher>() { @Override protected boolean matchesSafely(Bucket bucket) { - final PartFileWriter inProgressPart = bucket.getInProgressPart(); + final InProgressFileWriter inProgressPart = bucket.getInProgressPart(); return isNull == (inProgressPart == null); } @@ -349,23 +350,21 @@ public boolean supportsResume() { private static final RollingPolicy rollingPolicy = DefaultRollingPolicy.builder().build(); - private static final PartFileWriter.PartFileFactory partFileFactory = - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()); + private static final Encoder ENCODER = new SimpleStringEncoder<>(); private static Bucket createBucket( final RecoverableWriter writer, final Path bucketPath, final int subtaskIdx, final int initialPartCounter, - final OutputFileConfig outputFileConfig) { + final OutputFileConfig outputFileConfig) throws IOException { return Bucket.getNew( - writer, subtaskIdx, bucketId, bucketPath, initialPartCounter, - partFileFactory, + new RowWiseBucketWriter<>(writer, ENCODER), rollingPolicy, outputFileConfig); } @@ -378,10 +377,9 @@ private static Bucket restoreBucket( final OutputFileConfig outputFileConfig) throws Exception { return Bucket.restore( - writer, subtaskIndex, initialPartCounter, - partFileFactory, + new RowWiseBucketWriter<>(writer, ENCODER), rollingPolicy, bucketState, outputFileConfig); @@ -402,24 +400,46 @@ private static TestRecoverableWriter getRecoverableWriter(Path path) { private Bucket getRestoredBucketWithOnlyInProgressPart(final BaseStubWriter writer) throws IOException { final BucketState stateWithOnlyInProgressFile = - new BucketState<>("test", new Path(), 12345L, new NoOpRecoverable(), new HashMap<>()); - return Bucket.restore(writer, 0, 1L, partFileFactory, rollingPolicy, stateWithOnlyInProgressFile, OutputFileConfig.builder().build()); + new BucketState<>( + "test", + new Path(), + 12345L, + new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable(new NoOpRecoverable()), + new HashMap<>()); + + return Bucket.restore( + 0, + 1L, + new RowWiseBucketWriter<>(writer, ENCODER), + rollingPolicy, + stateWithOnlyInProgressFile, + OutputFileConfig.builder().build()); } private Bucket getRestoredBucketWithOnlyPendingParts(final BaseStubWriter writer, final int numberOfPendingParts) throws IOException { - final Map> completePartsPerCheckpoint = + final Map> completePartsPerCheckpoint = createPendingPartsPerCheckpoint(numberOfPendingParts); final BucketState initStateWithOnlyInProgressFile = - new BucketState<>("test", new Path(), 12345L, null, completePartsPerCheckpoint); - return Bucket.restore(writer, 0, 1L, partFileFactory, rollingPolicy, initStateWithOnlyInProgressFile, OutputFileConfig.builder().build()); + new BucketState<>( + "test", + new Path(), + 12345L, + null, + completePartsPerCheckpoint); + return Bucket.restore( + 0, + 1L, + new RowWiseBucketWriter<>(writer, ENCODER), + rollingPolicy, + initStateWithOnlyInProgressFile, OutputFileConfig.builder().build()); } - private Map> createPendingPartsPerCheckpoint(int noOfCheckpoints) { - final Map> pendingCommittablesPerCheckpoint = new HashMap<>(); + private Map> createPendingPartsPerCheckpoint(int noOfCheckpoints) { + final Map> pendingCommittablesPerCheckpoint = new HashMap<>(); for (int checkpointId = 0; checkpointId < noOfCheckpoints; checkpointId++) { - final List pending = new ArrayList<>(); - pending.add(new NoOpRecoverable()); + final List pending = new ArrayList<>(); + pending.add(new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(new NoOpRecoverable())); pendingCommittablesPerCheckpoint.put((long) checkpointId, pending); } return pendingCommittablesPerCheckpoint; diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java index e996444695887..8e2117a5f5447 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketsTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.serialization.SimpleStringEncoder; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.streaming.api.functions.sink.filesystem.TestUtils.MockListState; @@ -93,8 +94,8 @@ protected boolean matchesSafely(Bucket bucket) { return bucket.getBucketId().equals(bucketId) && bucket.getBucketPath().equals(new Path(testTmpPath, bucketId)) && bucket.getInProgressPart() == null && - bucket.getPendingPartsForCurrentCheckpoint().isEmpty() && - bucket.getPendingPartsPerCheckpoint().size() == 1; + bucket.getPendingFileRecoverablesForCurrentCheckpoint().isEmpty() && + bucket.getPendingFileRecoverablesPerCheckpoint().size() == 1; } @Override @@ -145,7 +146,7 @@ public void testMergeAtScaleInAndMaxCounterAtRecovery() throws Exception { Assert.assertEquals(2L, bucketsTwo.getMaxPartCounter()); // make sure we have one in-progress file here and a pending - Assert.assertEquals(1L, bucketsTwo.getActiveBuckets().get("test1").getPendingPartsPerCheckpoint().size()); + Assert.assertEquals(1L, bucketsTwo.getActiveBuckets().get("test1").getPendingFileRecoverablesPerCheckpoint().size()); Assert.assertNotNull(bucketsTwo.getActiveBuckets().get("test1").getInProgressPart()); final ListState mergedBucketStateContainer = new MockListState<>(); @@ -175,10 +176,10 @@ public void testMergeAtScaleInAndMaxCounterAtRecovery() throws Exception { // this is due to the Bucket#merge(). The in progress file of one // of the previous tasks is put in the list of pending files. - Assert.assertEquals(1L, bucket.getPendingPartsForCurrentCheckpoint().size()); + Assert.assertEquals(1L, bucket.getPendingFileRecoverablesForCurrentCheckpoint().size()); // we commit the pending for previous checkpoints - Assert.assertTrue(bucket.getPendingPartsPerCheckpoint().isEmpty()); + Assert.assertTrue(bucket.getPendingFileRecoverablesPerCheckpoint().isEmpty()); } @Test @@ -210,8 +211,8 @@ public void testOnProcessingTime() throws Exception { Assert.assertEquals("test", bucket.getBucketId()); Assert.assertNull(bucket.getInProgressPart()); - Assert.assertEquals(1L, bucket.getPendingPartsForCurrentCheckpoint().size()); - Assert.assertTrue(bucket.getPendingPartsPerCheckpoint().isEmpty()); + Assert.assertEquals(1L, bucket.getPendingFileRecoverablesForCurrentCheckpoint().size()); + Assert.assertTrue(bucket.getPendingFileRecoverablesPerCheckpoint().isEmpty()); } @Test @@ -321,7 +322,7 @@ private void testCorrectTimestampPassingInContext(Long timestamp, long watermark path, new VerifyingBucketAssigner(timestamp, watermark, processingTime), new DefaultBucketFactoryImpl<>(), - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + new RowWiseBucketWriter<>(FileSystem.get(path.toUri()).createRecoverableWriter(), new SimpleStringEncoder<>()), DefaultRollingPolicy.builder().build(), null, 2, @@ -458,7 +459,7 @@ private static Buckets createBuckets( basePath, new TestUtils.StringIdentityBucketAssigner(), new DefaultBucketFactoryImpl<>(), - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + new RowWiseBucketWriter<>(FileSystem.get(basePath.toUri()).createRecoverableWriter(), new SimpleStringEncoder<>()), rollingPolicy, bucketLifeCycleListener, subtaskIdx, diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicyTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicyTest.java index 59ea627359a98..2a4da34363631 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicyTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/RollingPolicyTest.java @@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.functions.sink.filesystem; import org.apache.flink.api.common.serialization.SimpleStringEncoder; +import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy; @@ -201,7 +202,7 @@ private static Buckets createBuckets( basePath, new TestUtils.StringIdentityBucketAssigner(), new DefaultBucketFactoryImpl<>(), - new RowWisePartWriter.Factory<>(new SimpleStringEncoder<>()), + new RowWiseBucketWriter<>(FileSystem.get(basePath.toUri()).createRecoverableWriter(), new SimpleStringEncoder<>()), rollingPolicyToTest, null, 0, diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java index c540dc7383317..df678c5d8a3ec 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java @@ -27,7 +27,6 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer; import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; -import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy; import org.apache.flink.streaming.api.operators.StreamSink; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; @@ -50,6 +49,8 @@ import java.util.List; import java.util.Map; +import static org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy.build; + /** * Utilities for the {@link StreamingFileSink} tests. */ @@ -158,7 +159,7 @@ static OneInputStreamOperatorTestHarness, Object> create .forBulkFormat(new Path(outDir.toURI()), writer) .withBucketAssigner(bucketer) .withBucketCheckInterval(bucketCheckInterval) - .withRollingPolicy(OnCheckpointRollingPolicy.build()) + .withRollingPolicy(build()) .withBucketFactory(bucketFactory) .withOutputFileConfig(outputFileConfig) .build(); @@ -199,7 +200,7 @@ static OneInputStreamOperatorTestHarness, Object> c StreamingFileSink> sink = StreamingFileSink .forBulkFormat(new Path(outDir.toURI()), writer) .withNewBucketAssigner(bucketer) - .withRollingPolicy(OnCheckpointRollingPolicy.build()) + .withRollingPolicy(build()) .withBucketCheckInterval(bucketCheckInterval) .withBucketFactory(bucketFactory) .withOutputFileConfig(outputFileConfig) diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java index e21da2aeabd73..6260a2c93169d 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/utils/NoOpRecoverableWriter.java @@ -50,7 +50,7 @@ public boolean requiresCleanupOfRecoverableState() { @Override public boolean cleanupRecoverableState(ResumeRecoverable resumable) throws IOException { - throw new UnsupportedOperationException(); + return false; } @Override From 4349e93eec94d9b89c21f473c770bc03adfd652a Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 18 May 2020 13:06:11 +0200 Subject: [PATCH 011/773] [FLINK-17593] Update BucketStateSerializerTest for v2 --- .../filesystem/BucketStateSerializerTest.java | 4 ++-- .../empty-v2/snapshot | Bin 0 -> 128 bytes ...progress.1e22e72d-0ab2-493b-8b00-9edac4252cec | 2 ++ ...progress.3821f491-9fa1-48b2-b66b-655352a3c8ec | 2 ++ ...progress.0af18f41-d8f8-4a4e-a92e-de12851be20b | 2 ++ ...progress.a3d0f4d2-d6ad-4f83-ba62-ed4b1fa86db2 | 2 ++ ...progress.666acf3e-935c-4621-8171-f7c897496524 | 2 ++ .../full-no-in-progress-v2-template/snapshot | Bin 0 -> 1597 bytes ...progress.9731063e-2b28-4701-8cc1-e706480b8022 | 2 ++ ...progress.1d423406-097a-4deb-bfde-d023d3477cd5 | 2 ++ ...progress.6a837aa3-4736-4098-a878-fdeffe227628 | 2 ++ ...progress.f121b73d-ac74-4fbd-b70d-f13e51c9132c | 2 ++ ...progress.a156884a-f090-4c3f-a271-0b63ab539c45 | 2 ++ ...progress.83c527c5-14dc-4d49-9f99-c915f2224f6a | 1 + .../full-v2-template/snapshot | Bin 0 -> 1685 bytes ...progress.10833090-dd8c-4e36-884d-bb9758a3a8ef | 1 + .../only-in-progress-v2/snapshot | Bin 0 -> 416 bytes 17 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v2/snapshot create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-0.inprogress.1e22e72d-0ab2-493b-8b00-9edac4252cec create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-1.inprogress.3821f491-9fa1-48b2-b66b-655352a3c8ec create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-2.inprogress.0af18f41-d8f8-4a4e-a92e-de12851be20b create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-3.inprogress.a3d0f4d2-d6ad-4f83-ba62-ed4b1fa86db2 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-4.inprogress.666acf3e-935c-4621-8171-f7c897496524 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/snapshot create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-0.inprogress.9731063e-2b28-4701-8cc1-e706480b8022 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-1.inprogress.1d423406-097a-4deb-bfde-d023d3477cd5 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-2.inprogress.6a837aa3-4736-4098-a878-fdeffe227628 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-3.inprogress.f121b73d-ac74-4fbd-b70d-f13e51c9132c create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-4.inprogress.a156884a-f090-4c3f-a271-0b63ab539c45 create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-5.inprogress.83c527c5-14dc-4d49-9f99-c915f2224f6a create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/full-v2-template/snapshot create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/bucket/test-bucket/.part-0-0.inprogress.10833090-dd8c-4e36-884d-bb9758a3a8ef create mode 100644 flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/snapshot diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java index cbd18c486dc38..bc6c1e53cdaeb 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateSerializerTest.java @@ -61,11 +61,11 @@ @RunWith(Parameterized.class) public class BucketStateSerializerTest { - private static final int CURRENT_VERSION = 1; + private static final int CURRENT_VERSION = 2; @Parameterized.Parameters(name = "Previous Version = {0}") public static Collection previousVersions() { - return Arrays.asList(1); + return Arrays.asList(1, 2); } @Parameterized.Parameter diff --git a/flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v2/snapshot b/flink-streaming-java/src/test/resources/bucket-state-migration-test/empty-v2/snapshot new file mode 100644 index 0000000000000000000000000000000000000000..9e84e8d9f93c0442a15d57bf1443a6519b4bc6b4 GIT binary patch literal 128 zcmZQzU|?ckV5pEQ^R5Kaj6lrK4J1lZi%WEqN|UowOBlS0i<0$00{TU%#rdU0$*INq w5Gmc_lEji!-Q3LdqQsKS{5)N-s?^+ql1kk&Bd8*r2KG~is-utW^dnoB(Gg-vwk3Jeb?+;>$~hU1zpt9!+W)9 zb*!2ij;q3cyM?-es)k+LY|^F<4mfN%9yFZ%`Flz!-ZAoZ-xyN5$>Q

-jtN`}Xb2 zIxg>rNuBDSl&88_w$p8d9P-Hmtj-ZuReeTUQIF_LbdYOWdM6aZis3RI#}Z_q75;?3CaJ^xAN09AKvSo^mc(RQjNQ;9WV!Gc?G15U*O z*y9WtDF;Y3mR{u8&a>Vp$3ln~LcIQY`E>JD2yxQl;%$2jr5_;f(w8!b>%L3tabVq* ze!J^Z>DT|KpdZ3e;C^%4g<-ROfc>hvySsxYoYw5WJfspQ-Mjwk{kY8-e}Dga^Kt6` z`Ft~6p0$^;eQNQo#bP_JA44~Q1if6w)gk3-^YGYhZyQd(a+EaMq(N~`z?nVx>OCZ{ zbVWq2CbYW7;$O8mv^~a$?^i$Cz}VkooKLgQh9Q^7kaG-gNl8NmFrovtQh`fJL9x=X zlhH}Usm8JB;~=Qz5!4((WkzCYKzbY6i6{|b%>}YV%EZ>DtS5x_Cmsm_mq)-kf(edn z2-FS%8DQxh1ZN#Ih*}GxHRvXUmq$aOF(P!+0ZDMLt>foseVt;|pr6|LG0L6|0l i)1x8q@(4Uf2ud60I6#$N0w<~v&?=D8P>7m*V*L-PLkcwj literal 0 HcmV?d00001 diff --git a/flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/bucket/test-bucket/.part-0-0.inprogress.10833090-dd8c-4e36-884d-bb9758a3a8ef b/flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/bucket/test-bucket/.part-0-0.inprogress.10833090-dd8c-4e36-884d-bb9758a3a8ef new file mode 100644 index 0000000000000..631ee76e88c1a --- /dev/null +++ b/flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/bucket/test-bucket/.part-0-0.inprogress.10833090-dd8c-4e36-884d-bb9758a3a8ef @@ -0,0 +1 @@ +writing diff --git a/flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/snapshot b/flink-streaming-java/src/test/resources/bucket-state-migration-test/only-in-progress-v2/snapshot new file mode 100644 index 0000000000000000000000000000000000000000..d21f7c46ccc436a46eba6968b505d8040d24ba9d GIT binary patch literal 416 zcmcIgOAdlC5S_Zw#29bD*4022)Uxmdtc^R90+kRINLw(R!mS7J242ts;RsIhCi8vs z0e}erdpS+i0vUwmoRn3@Vq7OzW#MRa;&lN}D^u1wQO5hF&{$~|-t$aLo0kRlRb_E& zF)y&vWkwwao6S()vy8OB^vbYDHpZD}I*z6QpXGTsZXTp>4mo2 literal 0 HcmV?d00001 From f46735cb4963af616c0e8538331bed8739a1d353 Mon Sep 17 00:00:00 2001 From: Gary Yao Date: Mon, 18 May 2020 10:08:49 +0200 Subject: [PATCH 012/773] [FLINK-17777][tests] Set HADOOP_CLASSPATH for Mesos TaskManagers --- flink-jepsen/src/jepsen/flink/db.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/flink-jepsen/src/jepsen/flink/db.clj b/flink-jepsen/src/jepsen/flink/db.clj index df047656e2949..61743b19a7258 100644 --- a/flink-jepsen/src/jepsen/flink/db.clj +++ b/flink-jepsen/src/jepsen/flink/db.clj @@ -303,6 +303,7 @@ "-Djobmanager.rpc.address=$(hostname -f)" "-Djobmanager.rpc.port=6123" "-Dmesos.resourcemanager.tasks.cpus=1" + "-Dcontainerized.taskmanager.env.HADOOP_CLASSPATH=$(/opt/hadoop/bin/hadoop classpath)" "-Dtaskmanager.memory.process.size=2048m" "-Drest.bind-address=$(hostname -f)")) From dea6dace89da6b7363a997667553c1da625dddd9 Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Mon, 18 May 2020 20:15:56 +0800 Subject: [PATCH 013/773] [hotfix][client] Make ConfigUtils.decodeListFromConfig return new array list and throw exception --- .../client/cli/ExecutionConfigAccessor.java | 19 ++++--------------- .../executors/EmbeddedExecutor.java | 5 +++-- .../deployment/executors/LocalExecutor.java | 3 ++- .../executors/PipelineExecutorUtils.java | 4 +++- ...ClassPathPackagedProgramRetrieverTest.java | 3 ++- .../flink/configuration/ConfigUtils.java | 18 ++++++++++++------ 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java index d1a93a8f9ab86..3894a92d79452 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/ExecutionConfigAccessor.java @@ -19,7 +19,6 @@ package org.apache.flink.client.cli; import org.apache.flink.annotation.Internal; -import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; @@ -72,22 +71,12 @@ public Configuration applyToConfiguration(final Configuration baseConfiguration) return baseConfiguration; } - public List getJars() { - return decodeUrlList(configuration, PipelineOptions.JARS); + public List getJars() throws MalformedURLException { + return ConfigUtils.decodeListFromConfig(configuration, PipelineOptions.JARS, URL::new); } - public List getClasspaths() { - return decodeUrlList(configuration, PipelineOptions.CLASSPATHS); - } - - private List decodeUrlList(final Configuration configuration, final ConfigOption> configOption) { - return ConfigUtils.decodeListFromConfig(configuration, configOption, url -> { - try { - return new URL(url); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("Invalid URL", e); - } - }); + public List getClasspaths() throws MalformedURLException { + return ConfigUtils.decodeListFromConfig(configuration, PipelineOptions.CLASSPATHS, URL::new); } public int getParallelism() { diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutor.java index 9c14e60171f08..febf3aeb1fdf1 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutor.java +++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutor.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; +import java.net.MalformedURLException; import java.util.Collection; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -81,7 +82,7 @@ public EmbeddedExecutor( } @Override - public CompletableFuture execute(final Pipeline pipeline, final Configuration configuration) { + public CompletableFuture execute(final Pipeline pipeline, final Configuration configuration) throws MalformedURLException { checkNotNull(pipeline); checkNotNull(configuration); @@ -101,7 +102,7 @@ private CompletableFuture getJobClientFuture(final JobID jobId) { return CompletableFuture.completedFuture(jobClientCreator.getJobClient(jobId)); } - private CompletableFuture submitAndGetJobClientFuture(final Pipeline pipeline, final Configuration configuration) { + private CompletableFuture submitAndGetJobClientFuture(final Pipeline pipeline, final Configuration configuration) throws MalformedURLException { final Time timeout = Time.milliseconds(configuration.get(ExecutionOptions.EMBEDDED_RPC_TIMEOUT).toMillis()); final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration); diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/LocalExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/LocalExecutor.java index 401ea194abddf..a64f03091aa97 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/LocalExecutor.java +++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/LocalExecutor.java @@ -32,6 +32,7 @@ import org.apache.flink.runtime.minicluster.MiniCluster; import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; +import java.net.MalformedURLException; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -80,7 +81,7 @@ public CompletableFuture execute(Pipeline pipeline, Configuration con return PerJobMiniClusterFactory.createWithFactory(effectiveConfig, miniClusterFactory).submitJob(jobGraph); } - private JobGraph getJobGraph(Pipeline pipeline, Configuration configuration) { + private JobGraph getJobGraph(Pipeline pipeline, Configuration configuration) throws MalformedURLException { // This is a quirk in how LocalEnvironment used to work. It sets the default parallelism // to * . Might be questionable but we keep the behaviour // for now. diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/PipelineExecutorUtils.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/PipelineExecutorUtils.java index 7fec53da5029c..0647091303d86 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/PipelineExecutorUtils.java +++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/PipelineExecutorUtils.java @@ -28,6 +28,8 @@ import javax.annotation.Nonnull; +import java.net.MalformedURLException; + import static org.apache.flink.util.Preconditions.checkNotNull; /** @@ -44,7 +46,7 @@ public class PipelineExecutorUtils { * savepoint settings used to bootstrap its state. * @return the corresponding {@link JobGraph}. */ - public static JobGraph getJobGraph(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration) { + public static JobGraph getJobGraph(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration) throws MalformedURLException { checkNotNull(pipeline); checkNotNull(configuration); diff --git a/flink-clients/src/test/java/org/apache/flink/client/deployment/application/ClassPathPackagedProgramRetrieverTest.java b/flink-clients/src/test/java/org/apache/flink/client/deployment/application/ClassPathPackagedProgramRetrieverTest.java index 41019188a483d..acdc6c2219410 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/deployment/application/ClassPathPackagedProgramRetrieverTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/deployment/application/ClassPathPackagedProgramRetrieverTest.java @@ -48,6 +48,7 @@ import java.io.File; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -328,7 +329,7 @@ public void testRetrieveFromJarFileWithUserLib() throws IOException, FlinkExcept containsInAnyOrder(expectedURLs.stream().map(URL::toString).toArray())); } - private JobGraph retrieveJobGraph(ClassPathPackagedProgramRetriever retrieverUnderTest, Configuration configuration) throws FlinkException, ProgramInvocationException { + private JobGraph retrieveJobGraph(ClassPathPackagedProgramRetriever retrieverUnderTest, Configuration configuration) throws FlinkException, ProgramInvocationException, MalformedURLException { final PackagedProgram packagedProgram = retrieverUnderTest.getPackagedProgram(); final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM); diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigUtils.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigUtils.java index bb7f4707678f1..117a3163113a4 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigUtils.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigUtils.java @@ -19,13 +19,13 @@ package org.apache.flink.configuration; import org.apache.flink.annotation.Internal; +import org.apache.flink.util.function.FunctionWithException; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -108,19 +108,25 @@ public static void encodeCollectionToConfig( * @param mapper the transformation function from {@code IN} to {@code OUT}. * @return the transformed values in a list of type {@code OUT}. */ - public static List decodeListFromConfig( + public static List decodeListFromConfig( final ReadableConfig configuration, final ConfigOption> key, - final Function mapper) { + final FunctionWithException mapper) throws E { checkNotNull(configuration); checkNotNull(key); checkNotNull(mapper); final List encodedString = configuration.get(key); - return encodedString != null - ? encodedString.stream().map(mapper).collect(Collectors.toList()) - : Collections.emptyList(); + if (encodedString == null || encodedString.isEmpty()) { + return new ArrayList<>(); + } + + final List result = new ArrayList<>(encodedString.size()); + for (IN input : encodedString) { + result.add(mapper.apply(input)); + } + return result; } private ConfigUtils() { From 59a657094194d2b1f290037c9404f12ee080566a Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Mon, 18 May 2020 18:29:48 +0800 Subject: [PATCH 014/773] [FLINK-17796] Respect user specified classpath for application mode --- .../ApplicationClusterEntryPoint.java | 25 +++++++++++++++++++ ...tandaloneApplicationClusterEntryPoint.java | 14 +++++------ ...ubernetesApplicationClusterEntrypoint.java | 14 +++++------ .../YarnApplicationClusterEntryPoint.java | 13 +++++----- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/application/ApplicationClusterEntryPoint.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/application/ApplicationClusterEntryPoint.java index aac0704666791..9dbc8fa3a8cd8 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/deployment/application/ApplicationClusterEntryPoint.java +++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/application/ApplicationClusterEntryPoint.java @@ -18,8 +18,12 @@ package org.apache.flink.client.deployment.application; +import org.apache.flink.client.deployment.application.executors.EmbeddedExecutor; import org.apache.flink.client.program.PackagedProgram; +import org.apache.flink.configuration.ConfigUtils; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.runtime.concurrent.ScheduledExecutor; import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore; import org.apache.flink.runtime.dispatcher.MemoryArchivedExecutionGraphStore; @@ -31,6 +35,12 @@ import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; import org.apache.flink.runtime.rest.JobRestEndpointFactory; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + import static org.apache.flink.util.Preconditions.checkNotNull; /** @@ -69,4 +79,19 @@ protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore( final ScheduledExecutor scheduledExecutor) { return new MemoryArchivedExecutionGraphStore(); } + + protected static void configureExecution(final Configuration configuration, final PackagedProgram program) throws MalformedURLException { + configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME); + ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.JARS, program.getJobJarAndDependencies(), URL::toString); + ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, getClasspath(configuration, program), URL::toString); + } + + private static List getClasspath(final Configuration configuration, final PackagedProgram program) throws MalformedURLException { + final List classpath = ConfigUtils.decodeListFromConfig( + configuration, + PipelineOptions.CLASSPATHS, + URL::new); + classpath.addAll(program.getClasspaths()); + return Collections.unmodifiableList(classpath.stream().distinct().collect(Collectors.toList())); + } } diff --git a/flink-container/src/main/java/org/apache/flink/container/entrypoint/StandaloneApplicationClusterEntryPoint.java b/flink-container/src/main/java/org/apache/flink/container/entrypoint/StandaloneApplicationClusterEntryPoint.java index 08894505973a1..41e874347e77d 100644 --- a/flink-container/src/main/java/org/apache/flink/container/entrypoint/StandaloneApplicationClusterEntryPoint.java +++ b/flink-container/src/main/java/org/apache/flink/container/entrypoint/StandaloneApplicationClusterEntryPoint.java @@ -23,13 +23,9 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.client.deployment.application.ApplicationClusterEntryPoint; import org.apache.flink.client.deployment.application.ClassPathPackagedProgramRetriever; -import org.apache.flink.client.deployment.application.executors.EmbeddedExecutor; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.client.program.PackagedProgramRetriever; -import org.apache.flink.configuration.ConfigUtils; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.DeploymentOptions; -import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.configuration.PipelineOptionsInternal; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.entrypoint.parser.CommandLineParser; @@ -44,7 +40,6 @@ import java.io.File; import java.io.IOException; -import java.net.URL; import static org.apache.flink.runtime.util.ClusterEntrypointUtils.tryFindUserLibDirectory; @@ -87,9 +82,12 @@ public static void main(String[] args) { } Configuration configuration = loadConfigurationFromClusterConfig(clusterConfiguration); - configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME); - ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.JARS, program.getJobJarAndDependencies(), URL::toString); - ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, program.getClasspaths(), URL::toString); + try { + configureExecution(configuration, program); + } catch (Exception e) { + LOG.error("Could not apply application configuration.", e); + System.exit(1); + } StandaloneApplicationClusterEntryPoint entrypoint = new StandaloneApplicationClusterEntryPoint(configuration, program); diff --git a/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/entrypoint/KubernetesApplicationClusterEntrypoint.java b/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/entrypoint/KubernetesApplicationClusterEntrypoint.java index de9023d33f882..1c4902ffb556a 100644 --- a/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/entrypoint/KubernetesApplicationClusterEntrypoint.java +++ b/flink-kubernetes/src/main/java/org/apache/flink/kubernetes/entrypoint/KubernetesApplicationClusterEntrypoint.java @@ -22,13 +22,9 @@ import org.apache.flink.client.deployment.application.ApplicationClusterEntryPoint; import org.apache.flink.client.deployment.application.ApplicationConfiguration; import org.apache.flink.client.deployment.application.ClassPathPackagedProgramRetriever; -import org.apache.flink.client.deployment.application.executors.EmbeddedExecutor; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.client.program.PackagedProgramRetriever; -import org.apache.flink.configuration.ConfigUtils; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.DeploymentOptions; -import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.kubernetes.utils.KubernetesUtils; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.util.EnvironmentInformation; @@ -41,7 +37,6 @@ import java.io.File; import java.io.IOException; -import java.net.URL; import java.util.List; import static org.apache.flink.runtime.util.ClusterEntrypointUtils.tryFindUserLibDirectory; @@ -74,9 +69,12 @@ public static void main(final String[] args) { System.exit(1); } - configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME); - ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.JARS, program.getJobJarAndDependencies(), URL::toString); - ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, program.getClasspaths(), URL::toString); + try { + configureExecution(configuration, program); + } catch (Exception e) { + LOG.error("Could not apply application configuration.", e); + System.exit(1); + } final KubernetesApplicationClusterEntrypoint kubernetesApplicationClusterEntrypoint = new KubernetesApplicationClusterEntrypoint(configuration, program); diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnApplicationClusterEntryPoint.java b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnApplicationClusterEntryPoint.java index 9f57b207cd602..c8b107cdf2e7a 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnApplicationClusterEntryPoint.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnApplicationClusterEntryPoint.java @@ -22,12 +22,9 @@ import org.apache.flink.client.deployment.application.ApplicationClusterEntryPoint; import org.apache.flink.client.deployment.application.ApplicationConfiguration; import org.apache.flink.client.deployment.application.ClassPathPackagedProgramRetriever; -import org.apache.flink.client.deployment.application.executors.EmbeddedExecutor; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.client.program.PackagedProgramRetriever; -import org.apache.flink.configuration.ConfigUtils; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.DeploymentOptions; import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.util.EnvironmentInformation; @@ -44,7 +41,6 @@ import java.io.File; import java.io.IOException; -import java.net.URL; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -96,9 +92,12 @@ public static void main(final String[] args) { System.exit(1); } - configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME); - ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.JARS, program.getJobJarAndDependencies(), URL::toString); - ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, program.getClasspaths(), URL::toString); + try { + configureExecution(configuration, program); + } catch (Exception e) { + LOG.error("Could not apply application configuration.", e); + System.exit(1); + } YarnApplicationClusterEntryPoint yarnApplicationClusterEntrypoint = new YarnApplicationClusterEntryPoint(configuration, program); From 656d56e99e3c158c7252db04bc034cce77ad39ba Mon Sep 17 00:00:00 2001 From: Gary Yao Date: Thu, 14 May 2020 17:10:35 +0200 Subject: [PATCH 015/773] [FLINK-17687][tests] Enable listing files recursively by pattern --- flink-jepsen/src/jepsen/flink/utils.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flink-jepsen/src/jepsen/flink/utils.clj b/flink-jepsen/src/jepsen/flink/utils.clj index 5d8e7126fa654..39ee36cb71f68 100644 --- a/flink-jepsen/src/jepsen/flink/utils.clj +++ b/flink-jepsen/src/jepsen/flink/utils.clj @@ -58,16 +58,17 @@ (defn find-files! "Lists files recursively given a directory. If the directory does not exist, an empty collection is returned." - [dir] + ([dir] (find-files! dir "*")) + ([dir name] (let [files (try - (c/exec :find dir :-type :f) + (c/exec :find dir :-type :f :-name (c/lit (str "\"" name "\""))) (catch Exception e (if (.contains (.getMessage e) "No such file or directory") "" (throw e))))] (->> (clojure.string/split files #"\n") - (remove clojure.string/blank?)))) + (remove clojure.string/blank?))))) ;;; runit process supervisor (http://smarden.org/runit/) From aa2a5709309ef8149607cc6ac696cd990a8aef81 Mon Sep 17 00:00:00 2001 From: Gary Yao Date: Thu, 14 May 2020 17:12:08 +0200 Subject: [PATCH 016/773] [FLINK-17687][tests] Collect log files before tearing down Mesos --- flink-jepsen/src/jepsen/flink/mesos.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flink-jepsen/src/jepsen/flink/mesos.clj b/flink-jepsen/src/jepsen/flink/mesos.clj index e944089c6a676..c104158f865d0 100644 --- a/flink-jepsen/src/jepsen/flink/mesos.clj +++ b/flink-jepsen/src/jepsen/flink/mesos.clj @@ -22,6 +22,7 @@ [util :as util :refer [meh]]] [jepsen.control.util :as cu] [jepsen.os.debian :as debian] + [jepsen.flink.utils :as fu] [jepsen.flink.utils :refer [create-supervised-service! stop-supervised-service!]] [jepsen.flink.zookeeper :refer [zookeeper-uri]])) @@ -195,4 +196,6 @@ (stop-marathon! test node)) db/LogFiles (log-files [_ test node] - (if (cu/exists? log-dir) (cu/ls-full log-dir) [])))) + (concat + (if (cu/exists? log-dir) (cu/ls-full log-dir) []) + (fu/find-files! slave-dir "*.log"))))) From 2aa45cdcc88d75837df165fcde71200d796deee7 Mon Sep 17 00:00:00 2001 From: Gary Yao Date: Thu, 14 May 2020 17:13:28 +0200 Subject: [PATCH 017/773] [FLINK-17687][tests] Simplify collection of Mesos logs --- flink-jepsen/src/jepsen/flink/mesos.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-jepsen/src/jepsen/flink/mesos.clj b/flink-jepsen/src/jepsen/flink/mesos.clj index c104158f865d0..de1ba50c1d9bc 100644 --- a/flink-jepsen/src/jepsen/flink/mesos.clj +++ b/flink-jepsen/src/jepsen/flink/mesos.clj @@ -197,5 +197,5 @@ db/LogFiles (log-files [_ test node] (concat - (if (cu/exists? log-dir) (cu/ls-full log-dir) []) + (fu/find-files! log-dir) (fu/find-files! slave-dir "*.log"))))) From 3424d660a3209fbbd34e644de29328001cf69013 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Mon, 4 May 2020 20:08:48 -0500 Subject: [PATCH 018/773] [hotfix][state-processor-api] Remove BoundedStreamConfig --- .../state/api/BootstrapTransformation.java | 17 ++++--- .../api/runtime/BoundedStreamConfig.java | 51 ------------------- 2 files changed, 11 insertions(+), 57 deletions(-) delete mode 100644 flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/BoundedStreamConfig.java diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java index 5e2a7c2ac8214..1c6127bc9d5a7 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java @@ -26,6 +26,7 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.operators.MapPartitionOperator; +import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.checkpoint.OperatorState; import org.apache.flink.runtime.jobgraph.OperatorID; @@ -36,7 +37,7 @@ import org.apache.flink.state.api.output.operators.BroadcastStateBootstrapOperator; import org.apache.flink.state.api.output.partitioner.HashSelector; import org.apache.flink.state.api.output.partitioner.KeyGroupRangePartitioner; -import org.apache.flink.state.api.runtime.BoundedStreamConfig; +import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.StreamOperator; @@ -175,12 +176,16 @@ MapPartitionOperator writeOperatorSubtaskStates( @VisibleForTesting StreamConfig getConfig(OperatorID operatorID, StateBackend stateBackend, StreamOperator operator) { - final StreamConfig config; - if (keyType == null) { - config = new BoundedStreamConfig(); - } else { + final StreamConfig config = new StreamConfig(new Configuration()); + config.setChainStart(); + config.setCheckpointingEnabled(true); + config.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE); + + if (keyType != null) { TypeSerializer keySerializer = keyType.createSerializer(dataSet.getExecutionEnvironment().getConfig()); - config = new BoundedStreamConfig(keySerializer, originalKeySelector); + + config.setStateKeySerializer(keySerializer); + config.setStatePartitioner(0, originalKeySelector); } config.setStreamOperator(operator); diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/BoundedStreamConfig.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/BoundedStreamConfig.java deleted file mode 100644 index 5f3b38b02e6bf..0000000000000 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/BoundedStreamConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.state.api.runtime; - -import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.java.functions.KeySelector; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.CheckpointingMode; -import org.apache.flink.streaming.api.graph.StreamConfig; - -/** - * A {@link StreamConfig} with default settings. - */ -@Internal -public class BoundedStreamConfig extends StreamConfig { - - private static final long serialVersionUID = 1L; - - public BoundedStreamConfig() { - super(new Configuration()); - - setChainStart(); - setCheckpointingEnabled(true); - setCheckpointMode(CheckpointingMode.EXACTLY_ONCE); - } - - public BoundedStreamConfig(TypeSerializer keySerializer, KeySelector keySelector) { - this(); - - setStateKeySerializer(keySerializer); - setStatePartitioner(0, keySelector); - } -} - From e564a3a7773b199bfa2f0b37d1459ae3430bc6a5 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Mon, 4 May 2020 20:13:48 -0500 Subject: [PATCH 019/773] [FLINK-17506][state-processor-api] SavepointEnvironment should honour 'io.tmp.dirs' property --- .../org/apache/flink/state/api/BootstrapTransformation.java | 3 +-- .../apache/flink/state/api/runtime/SavepointEnvironment.java | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java index 1c6127bc9d5a7..5587046b70b06 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java @@ -26,7 +26,6 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.operators.MapPartitionOperator; -import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.checkpoint.OperatorState; import org.apache.flink.runtime.jobgraph.OperatorID; @@ -176,7 +175,7 @@ MapPartitionOperator writeOperatorSubtaskStates( @VisibleForTesting StreamConfig getConfig(OperatorID operatorID, StateBackend stateBackend, StreamOperator operator) { - final StreamConfig config = new StreamConfig(new Configuration()); + final StreamConfig config = new StreamConfig(dataSet.getExecutionEnvironment().getConfiguration()); config.setChainStart(); config.setCheckpointingEnabled(true); config.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE); diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java index fca7a74ec6e71..915b6766c2bf5 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java @@ -24,6 +24,7 @@ import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.accumulators.AccumulatorRegistry; import org.apache.flink.runtime.broadcast.BroadcastVariableManager; @@ -104,7 +105,7 @@ private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, in this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID); this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState); - this.ioManager = new IOManagerAsync(); + this.ioManager = new IOManagerAsync(ConfigurationUtils.parseTempDirectories(configuration)); this.memoryManager = MemoryManager.forDefaultPageSize(64 * 1024 * 1024); this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID); } From 3bf6596b99e0d7760398acdfeccf7184d3ddd900 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Mon, 18 May 2020 01:32:49 +0200 Subject: [PATCH 020/773] [FLINK-17781][coordination] Use Scheduler and MainThreadExecutor in OperatorCoordinator.Context. This needs (unfortunately) a switch to lazy initialization of the context: - The Scheduler needs to be created before the OperatorCoordinator Context are created. One could do that by creating the Coordinators lazily after the Scheduler. - The Scheduler restores the savepoints as part of the scheduler creation, when the ExecutionGraph and the CheckpointCoordinator are created early in the constructor. - That means the OperatorCoordinator needs to exist (or an in placeholder component, here the OperatorCoordinatorHolder) needs to exist to accept the restored state. That brings us to a cyclic dependency: - OperatorCoordinator (context) needs Scheduler and MainThreadExecutor - Scheduler and MainThreadExecutor need constructed ExecutionGraph - ExecutionGraph needs CheckpointCoordinator - CheckpointCoordinator needs OperatorCoordinator Breaking the Cycle To break this cyclic dependency, this change introduces a form of lazy initialization: - We eagerly create the OperatorCoordinators so they exist for state restore - We provide an uninitialized context to them . When the Scheduler is started (after leadership is granted) we initialize the context with the (then readily constructed) Scheduler and MainThreadExecutor This closes #12225 --- .../executiongraph/ExecutionGraph.java | 9 +- .../executiongraph/ExecutionJobVertex.java | 41 ++- .../coordination/OperatorCoordinator.java | 10 + .../OperatorCoordinatorHolder.java | 236 ++++++++++++++++++ .../coordination/OperatorCoordinatorUtil.java | 61 ----- .../runtime/scheduler/SchedulerBase.java | 56 +++-- .../flink/runtime/scheduler/SchedulerNG.java | 10 + .../OperatorCoordinatorSchedulerTest.java | 11 +- 8 files changed, 328 insertions(+), 106 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java delete mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorUtil.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java index e348059f4c35a..b84a86f4c0efb 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java @@ -60,12 +60,11 @@ import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID; import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.jobgraph.ScheduleMode; import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup; import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; -import org.apache.flink.runtime.operators.coordination.OperatorCoordinator; +import org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder; import org.apache.flink.runtime.query.KvStateLocationRegistry; import org.apache.flink.runtime.scheduler.InternalFailuresListener; import org.apache.flink.runtime.scheduler.adapter.DefaultExecutionTopology; @@ -570,10 +569,10 @@ private ExecutionVertex[] collectExecutionVertices(List jobV private Collection buildOpCoordinatorCheckpointContexts() { final ArrayList contexts = new ArrayList<>(); for (final ExecutionJobVertex vertex : verticesInCreationOrder) { - for (final Map.Entry coordinator : vertex.getOperatorCoordinatorMap().entrySet()) { + for (final OperatorCoordinatorHolder coordinator : vertex.getOperatorCoordinators()) { contexts.add(new OperatorCoordinatorCheckpointContext( - coordinator.getValue(), - coordinator.getKey(), + coordinator, + coordinator.getOperatorId(), vertex.getMaxParallelism(), vertex.getParallelism())); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java index ee2cd2a7ae58f..0b5f0ffeb9872 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java @@ -44,13 +44,13 @@ import org.apache.flink.runtime.jobgraph.JobEdge; import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup; import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; import org.apache.flink.runtime.operators.coordination.OperatorCoordinator; -import org.apache.flink.runtime.operators.coordination.OperatorCoordinatorUtil; +import org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.types.Either; +import org.apache.flink.util.IOUtils; import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; @@ -119,7 +119,7 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable */ private Either, PermanentBlobKey> taskInformationOrBlobKey = null; - private final Map operatorCoordinators; + private final Collection operatorCoordinators; private InputSplitAssigner splitAssigner; @@ -229,16 +229,20 @@ public ExecutionJobVertex( } } - try { - final Map coordinators = OperatorCoordinatorUtil.instantiateCoordinators( - jobVertex.getOperatorCoordinators(), - graph.getUserClassLoader(), - (opId) -> new ExecutionJobVertexCoordinatorContext(opId, this)); - - this.operatorCoordinators = Collections.unmodifiableMap(coordinators); - } - catch (IOException | ClassNotFoundException e) { - throw new JobException("Cannot instantiate the coordinator for operator " + getName(), e); + final List> coordinatorProviders = getJobVertex().getOperatorCoordinators(); + if (coordinatorProviders.isEmpty()) { + this.operatorCoordinators = Collections.emptyList(); + } else { + final ArrayList coordinators = new ArrayList<>(coordinatorProviders.size()); + try { + for (final SerializedValue provider : coordinatorProviders) { + coordinators.add(OperatorCoordinatorHolder.create(provider, this, graph.getUserClassLoader())); + } + } catch (Exception | LinkageError e) { + IOUtils.closeAllQuietly(coordinators); + throw new JobException("Cannot instantiate the coordinator for operator " + getName(), e); + } + this.operatorCoordinators = Collections.unmodifiableList(coordinators); } // set up the input splits, if the vertex has any @@ -371,19 +375,10 @@ public InputDependencyConstraint getInputDependencyConstraint() { return getJobVertex().getInputDependencyConstraint(); } - @Nullable - public OperatorCoordinator getOperatorCoordinator(OperatorID operatorId) { - return operatorCoordinators.get(operatorId); - } - - public Map getOperatorCoordinatorMap() { + public Collection getOperatorCoordinators() { return operatorCoordinators; } - public Collection getOperatorCoordinators() { - return operatorCoordinators.values(); - } - public Either, PermanentBlobKey> getTaskInformationOrBlobKey() throws IOException { // only one thread should offload the task information, so let's also let only one thread // serialize the task information! diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java index 27e09108651e3..cb388b2a05d77 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java @@ -32,6 +32,16 @@ * *

Operator coordinators are for example source and sink coordinators that discover and assign * work, or aggregate and commit metadata. + * + *

Thread Model

+ * + *

All coordinator methods are called by the Job Manager's main thread (mailbox thread). That means that + * these methods must not, under any circumstances, perform blocking operations (like I/O or waiting on + * locks or futures). That would run a high risk of bringing down the entire JobManager. + * + *

Coordinators that involve more complex operations should hence spawn threads to handle the I/O work. + * The methods on the {@link Context} are safe to be called from another thread than the thread that + * calls the Coordinator's methods. */ public interface OperatorCoordinator extends AutoCloseable { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java new file mode 100644 index 0000000000000..43c47fed1eccd --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.operators.coordination; + +import org.apache.flink.runtime.executiongraph.Execution; +import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; +import org.apache.flink.runtime.jobgraph.OperatorID; +import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.scheduler.SchedulerNG; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.SerializedValue; +import org.apache.flink.util.TemporaryClassLoaderContext; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A holder for an {@link OperatorCoordinator.Context} and all the necessary facility around it that + * is needed to interaction between the Coordinator, the Scheduler, the Checkpoint Coordinator, etc. + * + *

The holder is itself a {@link OperatorCoordinator} and forwards all calls to the actual coordinator. + * That way, we can make adjustments to assumptions about the threading model and message/call forwarding + * without needing to adjust all the call sites that interact with the coordinator. + * + *

This is also needed, unfortunately, because we need a lazy two-step initialization: + * When the execution graph is created, we need to create the coordinators (or the holders, to be specific) + * because the CheckpointCoordinator is also created in the ExecutionGraph and needs access to them. + * However, the real Coordinators can only be created after SchedulerNG was created, because they need + * a reference to it for the failure calls. + */ +public class OperatorCoordinatorHolder implements OperatorCoordinator { + + private final OperatorCoordinator coordinator; + private final OperatorID operatorId; + private final LazyInitializedCoordinatorContext context; + + private OperatorCoordinatorHolder( + final OperatorID operatorId, + final OperatorCoordinator coordinator, + final LazyInitializedCoordinatorContext context) { + + this.operatorId = checkNotNull(operatorId); + this.coordinator = checkNotNull(coordinator); + this.context = checkNotNull(context); + } + + // ------------------------------------------------------------------------ + + public OperatorID getOperatorId() { + return operatorId; + } + + public OperatorCoordinator getCoordinator() { + return coordinator; + } + + public void lazyInitialize(SchedulerNG scheduler, Executor schedulerExecutor) { + context.lazyInitialize(scheduler, schedulerExecutor); + } + + // ------------------------------------------------------------------------ + // OperatorCoordinator Interface + // ------------------------------------------------------------------------ + + @Override + public void start() throws Exception { + checkState(context.isInitialized(), "Coordinator Context is not yet initialized"); + coordinator.start(); + } + + @Override + public void close() throws Exception { + coordinator.close(); + context.unInitialize(); + } + + @Override + public void handleEventFromOperator(int subtask, OperatorEvent event) throws Exception { + coordinator.handleEventFromOperator(subtask, event); + } + + @Override + public void subtaskFailed(int subtask, @Nullable Throwable reason) { + coordinator.subtaskFailed(subtask, reason); + } + + @Override + public CompletableFuture checkpointCoordinator(long checkpointId) throws Exception { + return coordinator.checkpointCoordinator(checkpointId); + } + + @Override + public void checkpointComplete(long checkpointId) { + coordinator.checkpointComplete(checkpointId); + } + + @Override + public void resetToCheckpoint(byte[] checkpointData) throws Exception { + coordinator.resetToCheckpoint(checkpointData); + } + + // ------------------------------------------------------------------------ + // Factories + // ------------------------------------------------------------------------ + + public static OperatorCoordinatorHolder create( + SerializedValue serializedProvider, + ExecutionJobVertex jobVertex, + ClassLoader classLoader) throws IOException, ClassNotFoundException { + + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(classLoader)) { + final OperatorCoordinator.Provider provider = serializedProvider.deserializeValue(classLoader); + final OperatorID opId = provider.getOperatorId(); + final LazyInitializedCoordinatorContext context = new LazyInitializedCoordinatorContext(opId, jobVertex); + final OperatorCoordinator coordinator = provider.create(context); + return new OperatorCoordinatorHolder(opId, coordinator, context); + } + } + + // ------------------------------------------------------------------------ + // Nested Classes + // ------------------------------------------------------------------------ + + /** + * An implementation of the {@link OperatorCoordinator.Context}. + * + *

All methods are safe to be called from other threads than the Scheduler's and the JobMaster's + * main threads. + * + *

Implementation note: Ideally, we would like to operate purely against the scheduler + * interface, but it is not exposing enough information at the moment. + */ + private static final class LazyInitializedCoordinatorContext implements OperatorCoordinator.Context { + + private final OperatorID operatorId; + private final ExecutionJobVertex jobVertex; + + private SchedulerNG scheduler; + private Executor schedulerExecutor; + + public LazyInitializedCoordinatorContext(OperatorID operatorId, ExecutionJobVertex jobVertex) { + this.operatorId = checkNotNull(operatorId); + this.jobVertex = checkNotNull(jobVertex); + } + + void lazyInitialize(SchedulerNG scheduler, Executor schedulerExecutor) { + this.scheduler = checkNotNull(scheduler); + this.schedulerExecutor = checkNotNull(schedulerExecutor); + } + + void unInitialize() { + this.scheduler = null; + this.schedulerExecutor = null; + } + + boolean isInitialized() { + return jobVertex != null; + } + + private void checkInitialized() { + checkState(isInitialized(), "Context was not yet initialized"); + } + + @Override + public OperatorID getOperatorId() { + return operatorId; + } + + @Override + public CompletableFuture sendEvent(final OperatorEvent evt, final int targetSubtask) { + checkInitialized(); + + if (targetSubtask < 0 || targetSubtask >= currentParallelism()) { + throw new IllegalArgumentException( + String.format("subtask index %d out of bounds [0, %d).", targetSubtask, currentParallelism())); + } + + final SerializedValue serializedEvent; + try { + serializedEvent = new SerializedValue<>(evt); + } + catch (IOException e) { + // we do not expect that this exception is handled by the caller, so we make it + // unchecked so that it can bubble up + throw new FlinkRuntimeException("Cannot serialize operator event", e); + } + + final Execution executionAttempt = jobVertex.getTaskVertices()[targetSubtask].getCurrentExecutionAttempt(); + return executionAttempt.sendOperatorEvent(operatorId, serializedEvent); + } + + @Override + public void failTask(final int subtask, final Throwable cause) { + throw new UnsupportedOperationException(); + } + + @Override + public void failJob(final Throwable cause) { + checkInitialized(); + + final FlinkException e = new FlinkException("Global failure triggered by OperatorCoordinator for '" + + jobVertex.getName() + "' (operator " + operatorId + ").", cause); + + schedulerExecutor.execute(() -> scheduler.handleGlobalFailure(e)); + } + + @Override + public int currentParallelism() { + checkInitialized(); + return jobVertex.getParallelism(); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorUtil.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorUtil.java deleted file mode 100644 index 154b547caf787..0000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorUtil.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.operators.coordination; - -import org.apache.flink.runtime.jobgraph.OperatorID; -import org.apache.flink.util.SerializedValue; -import org.apache.flink.util.TemporaryClassLoaderContext; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -/** - * A utility to for dealing with the {@link OperatorCoordinator}. - */ -public final class OperatorCoordinatorUtil { - - public static Map instantiateCoordinators( - List> providers, - ClassLoader classLoader, - Function contextFactory) throws IOException, ClassNotFoundException { - - try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(classLoader)) { - - final HashMap coordinators = new HashMap<>(); - - for (SerializedValue serializedProvider : providers) { - final OperatorCoordinator.Provider provider = serializedProvider.deserializeValue(classLoader); - final OperatorID id = provider.getOperatorId(); - final OperatorCoordinator.Context context = contextFactory.apply(id); - final OperatorCoordinator coordinator = provider.create(context); - coordinators.put(id, coordinator); - } - - return coordinators; - } - } - - // ------------------------------------------------------------------------ - - /** Utility class, not meant to be instantiated. */ - private OperatorCoordinatorUtil() {} -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java index c2c911a89113b..6f2ae478ba12e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java @@ -81,6 +81,7 @@ import org.apache.flink.runtime.operators.coordination.CoordinationRequestHandler; import org.apache.flink.runtime.operators.coordination.CoordinationResponse; import org.apache.flink.runtime.operators.coordination.OperatorCoordinator; +import org.apache.flink.runtime.operators.coordination.OperatorCoordinatorHolder; import org.apache.flink.runtime.operators.coordination.OperatorEvent; import org.apache.flink.runtime.operators.coordination.TaskNotRunningException; import org.apache.flink.runtime.query.KvStateLocation; @@ -169,7 +170,7 @@ public abstract class SchedulerBase implements SchedulerNG { protected final ExecutionVertexVersioner executionVertexVersioner; - private final Map coordinatorMap; + private final Map coordinatorMap; private ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor( "SchedulerBase is not initialized with proper main thread executor. " + @@ -443,6 +444,7 @@ CheckpointCoordinator getCheckpointCoordinator() { @Override public void setMainThreadExecutor(final ComponentMainThreadExecutor mainThreadExecutor) { this.mainThreadExecutor = checkNotNull(mainThreadExecutor); + initializeOperatorCoordinators(mainThreadExecutor); executionGraph.start(mainThreadExecutor); } @@ -901,6 +903,21 @@ private String retrieveTaskManagerLocation(ExecutionAttemptID executionAttemptID .orElse("Unknown location"); } + // ------------------------------------------------------------------------ + // Operator Coordinators + // + // Note: It may be worthwhile to move the OperatorCoordinators out + // of the scheduler (have them owned by the JobMaster directly). + // Then we could avoid routing these events through the scheduler and + // doing this lazy initialization dance. However, this would require + // that the Scheduler does not eagerly construct the CheckpointCoordinator + // in the ExecutionGraph and does not eagerly restore the savepoint while + // doing that. Because during savepoint restore, the OperatorCoordinators + // (or at least their holders) already need to exist, to accept the restored + // state. But some components they depend on (Scheduler and MainThreadExecutor) + // are not fully usable and accessible at that point. + // ------------------------------------------------------------------------ + @Override public void deliverOperatorEventToCoordinator( final ExecutionAttemptID taskExecutionId, @@ -922,8 +939,7 @@ public void deliverOperatorEventToCoordinator( throw new TaskNotRunningException("Task is not known or in state running on the JobManager."); } - final ExecutionJobVertex ejv = exec.getVertex().getJobVertex(); - final OperatorCoordinator coordinator = ejv.getOperatorCoordinator(operatorId); + final OperatorCoordinatorHolder coordinator = coordinatorMap.get(operatorId); if (coordinator == null) { throw new FlinkException("No coordinator registered for operator " + operatorId); } @@ -932,7 +948,7 @@ public void deliverOperatorEventToCoordinator( coordinator.handleEventFromOperator(exec.getParallelSubtaskIndex(), evt); } catch (Throwable t) { ExceptionUtils.rethrowIfFatalErrorOrOOM(t); - failJob(t); + handleGlobalFailure(t); } } @@ -940,20 +956,30 @@ public void deliverOperatorEventToCoordinator( public CompletableFuture deliverCoordinationRequestToCoordinator( OperatorID operator, CoordinationRequest request) throws FlinkException { - OperatorCoordinator coordinator = coordinatorMap.get(operator); + + final OperatorCoordinatorHolder coordinatorHolder = coordinatorMap.get(operator); + if (coordinatorHolder == null){ + throw new FlinkException("Coordinator of operator " + operator + " does not exist"); + } + + final OperatorCoordinator coordinator = coordinatorHolder.getCoordinator(); if (coordinator instanceof CoordinationRequestHandler) { return ((CoordinationRequestHandler) coordinator).handleCoordinationRequest(request); - } else if (coordinator != null) { - throw new FlinkException("Coordinator of operator " + operator + " cannot handle client event"); } else { - throw new FlinkException("Coordinator of operator " + operator + " does not exist"); + throw new FlinkException("Coordinator of operator " + operator + " cannot handle client event"); + } + } + + private void initializeOperatorCoordinators(Executor mainThreadExecutor) { + for (OperatorCoordinatorHolder coordinatorHolder : getAllCoordinators()) { + coordinatorHolder.lazyInitialize(this, mainThreadExecutor); } } private void startAllOperatorCoordinators() { - final Collection coordinators = getAllCoordinators(); + final Collection coordinators = getAllCoordinators(); try { - for (OperatorCoordinator coordinator : coordinators) { + for (OperatorCoordinatorHolder coordinator : coordinators) { coordinator.start(); } } @@ -968,15 +994,15 @@ private void disposeAllOperatorCoordinators() { getAllCoordinators().forEach(IOUtils::closeQuietly); } - private Collection getAllCoordinators() { + private Collection getAllCoordinators() { return coordinatorMap.values(); } - private Map createCoordinatorMap() { - Map coordinatorMap = new HashMap<>(); + private Map createCoordinatorMap() { + Map coordinatorMap = new HashMap<>(); for (ExecutionJobVertex vertex : executionGraph.getAllVertices().values()) { - for (Map.Entry entry : vertex.getOperatorCoordinatorMap().entrySet()) { - coordinatorMap.put(entry.getKey(), entry.getValue()); + for (OperatorCoordinatorHolder holder : vertex.getOperatorCoordinators()) { + coordinatorMap.put(holder.getOperatorId(), holder); } } return coordinatorMap; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerNG.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerNG.java index 07f7e7e008933..1675bc0acd7a8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerNG.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerNG.java @@ -126,6 +126,16 @@ public interface SchedulerNG { CompletableFuture stopWithSavepoint(String targetDirectory, boolean advanceToEndOfEventTime); + // ------------------------------------------------------------------------ + // Operator Coordinator related methods + // + // These are necessary as long as the Operator Coordinators are part of the + // scheduler. There are good reasons to pull them out of the Scheduler and + // make them directly a part of the JobMaster. However, we would need to + // rework the complete CheckpointCoordinator initialization before we can + // do that, because the CheckpointCoordinator is initialized (and restores + // savepoint) in the scheduler constructor, which requires the coordinators + // to be there as well. // ------------------------------------------------------------------------ /** diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java index 935e0ec26c2fe..14a136ddb5523 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java @@ -61,6 +61,7 @@ import java.io.IOException; import java.time.Duration; import java.util.Collections; +import java.util.Optional; import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; @@ -501,8 +502,14 @@ private TestingOperatorCoordinator getCoordinator(DefaultScheduler scheduler) { final ExecutionJobVertex vertexWithCoordinator = getJobVertex(scheduler, testVertexId); assertNotNull("vertex for coordinator not found", vertexWithCoordinator); - final OperatorCoordinator coordinator = vertexWithCoordinator.getOperatorCoordinator(testOperatorId); - assertNotNull("vertex does not contain coordinator", coordinator); + final Optional coordinatorOptional = vertexWithCoordinator + .getOperatorCoordinators() + .stream() + .filter((holder) -> holder.getOperatorId().equals(testOperatorId)) + .findFirst(); + assertTrue("vertex does not contain coordinator", coordinatorOptional.isPresent()); + + final OperatorCoordinator coordinator = coordinatorOptional.get().getCoordinator(); assertThat(coordinator, instanceOf(TestingOperatorCoordinator.class)); return (TestingOperatorCoordinator) coordinator; From b834ff6b2fe4cbdddd5da3345670f21ef5822d8d Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Mon, 18 May 2020 15:59:08 +0200 Subject: [PATCH 021/773] [FLINK-17799][network] Fix performance regression in the benchmarks FLINK-16536 (re) introduced requestPartitions on critical path of AbstractRecordReader, which amounts to couple of viritual calls, one lock acquisition and one volatile read. This overhead is what's cuasing performance drop. We can avoid subsequentional redundant calls, by checking against a local variable if we have already requested partitions or not. --- .../io/network/api/reader/AbstractRecordReader.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java index 6975ac6f1b262..1c98d0c3555e4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java @@ -41,6 +41,8 @@ abstract class AbstractRecordReader extends Abstra private RecordDeserializer currentRecordDeserializer; + private boolean requestedPartitions; + private boolean isFinished; /** @@ -66,7 +68,10 @@ protected boolean getNextRecord(T target) throws IOException, InterruptedExcepti // The action of partition request was removed from InputGate#setup since FLINK-16536, and this is the only // unified way for launching partition request for batch jobs. In order to avoid potential performance concern, // we might consider migrating this action back to the setup based on some condition judgement future. - inputGate.requestPartitions(); + if (!requestedPartitions) { + inputGate.requestPartitions(); + requestedPartitions = true; + } if (isFinished) { return false; From ce10080de0e6c35d60147318381c096cf77ec252 Mon Sep 17 00:00:00 2001 From: Seth Wiesman Date: Mon, 18 May 2020 13:47:09 -0500 Subject: [PATCH 022/773] [FLINK-17506][state-processor-api] Use proper RocksDB configurations in KeyedStateInputFormat --- .../flink/state/api/BootstrapTransformation.java | 9 ++++++--- .../apache/flink/state/api/ExistingSavepoint.java | 1 + .../state/api/input/KeyedStateInputFormat.java | 10 ++++++++++ .../state/api/input/KeyedStateInputFormatTest.java | 13 +++++++------ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java index 5587046b70b06..474a403e6b4cc 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/BootstrapTransformation.java @@ -26,6 +26,7 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.operators.MapPartitionOperator; +import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.checkpoint.OperatorState; import org.apache.flink.runtime.jobgraph.OperatorID; @@ -155,8 +156,7 @@ MapPartitionOperator writeOperatorSubtaskStates( BoundedOneInputStreamTaskRunner operatorRunner = new BoundedOneInputStreamTaskRunner<>( config, - localMaxParallelism - ); + localMaxParallelism); MapPartitionOperator subtaskStates = input .mapPartition(operatorRunner) @@ -175,7 +175,10 @@ MapPartitionOperator writeOperatorSubtaskStates( @VisibleForTesting StreamConfig getConfig(OperatorID operatorID, StateBackend stateBackend, StreamOperator operator) { - final StreamConfig config = new StreamConfig(dataSet.getExecutionEnvironment().getConfiguration()); + // Eagerly perform a deep copy of the configuration, otherwise it will result in undefined behavior + // when deploying with multiple bootstrap transformations. + Configuration deepCopy = new Configuration(dataSet.getExecutionEnvironment().getConfiguration()); + final StreamConfig config = new StreamConfig(deepCopy); config.setChainStart(); config.setCheckpointingEnabled(true); config.setCheckpointMode(CheckpointingMode.EXACTLY_ONCE); diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/ExistingSavepoint.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/ExistingSavepoint.java index 6c91660e26ec1..8bc639fd615e3 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/ExistingSavepoint.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/ExistingSavepoint.java @@ -280,6 +280,7 @@ public DataSet readKeyedState( KeyedStateInputFormat inputFormat = new KeyedStateInputFormat<>( operatorState, stateBackend, + env.getConfiguration(), new KeyedStateReaderOperator<>(function, keyTypeInfo)); return env.createInput(inputFormat, outTypeInfo); diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/input/KeyedStateInputFormat.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/input/KeyedStateInputFormat.java index 32df221be1afb..5fd91d8193819 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/input/KeyedStateInputFormat.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/input/KeyedStateInputFormat.java @@ -69,6 +69,8 @@ public class KeyedStateInputFormat extends RichInputFormat operator; private transient CloseableRegistry registry; @@ -82,17 +84,24 @@ public class KeyedStateInputFormat extends RichInputFormat operator) { Preconditions.checkNotNull(operatorState, "The operator state cannot be null"); Preconditions.checkNotNull(stateBackend, "The state backend cannot be null"); + Preconditions.checkNotNull(configuration, "The configuration cannot be null"); Preconditions.checkNotNull(operator, "The operator cannot be null"); this.operatorState = operatorState; this.stateBackend = stateBackend; + // Eagerly deep copy the configuration object + // otherwise there will be undefined behavior + // when executing pipelines with multiple input formats + this.configuration = new Configuration(configuration); this.operator = operator; } @@ -138,6 +147,7 @@ public void open(KeyGroupRangeInputSplit split) throws IOException { final Environment environment = new SavepointEnvironment .Builder(getRuntimeContext(), split.getNumKeyGroups()) + .setConfiguration(configuration) .setSubtaskIndex(split.getSplitNumber()) .setPrioritizedOperatorSubtaskState(split.getPrioritizedOperatorSubtaskState()) .build(); diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java index d0b55b6735630..441b9bf1265ca 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java @@ -67,7 +67,7 @@ public void testCreatePartitionedInputSplits() throws Exception { OperatorState operatorState = new OperatorState(operatorID, 1, 128); operatorState.putState(0, state); - KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); + KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); KeyGroupRangeInputSplit[] splits = format.createInputSplits(4); Assert.assertEquals("Failed to properly partition operator state into input splits", 4, splits.length); } @@ -80,7 +80,7 @@ public void testMaxParallelismRespected() throws Exception { OperatorState operatorState = new OperatorState(operatorID, 1, 128); operatorState.putState(0, state); - KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); + KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); KeyGroupRangeInputSplit[] splits = format.createInputSplits(129); Assert.assertEquals("Failed to properly partition operator state into input splits", 128, splits.length); } @@ -93,7 +93,7 @@ public void testReadState() throws Exception { OperatorState operatorState = new OperatorState(operatorID, 1, 128); operatorState.putState(0, state); - KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); + KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); KeyGroupRangeInputSplit split = format.createInputSplits(1)[0]; KeyedStateReaderFunction userFunction = new ReaderFunction(); @@ -111,7 +111,7 @@ public void testReadMultipleOutputPerKey() throws Exception { OperatorState operatorState = new OperatorState(operatorID, 1, 128); operatorState.putState(0, state); - KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); + KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); KeyGroupRangeInputSplit split = format.createInputSplits(1)[0]; KeyedStateReaderFunction userFunction = new DoubleReaderFunction(); @@ -129,7 +129,7 @@ public void testInvalidProcessReaderFunctionFails() throws Exception { OperatorState operatorState = new OperatorState(operatorID, 1, 128); operatorState.putState(0, state); - KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); + KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT)); KeyGroupRangeInputSplit split = format.createInputSplits(1)[0]; KeyedStateReaderFunction userFunction = new InvalidReaderFunction(); @@ -147,7 +147,7 @@ public void testReadTime() throws Exception { OperatorState operatorState = new OperatorState(operatorID, 1, 128); operatorState.putState(0, state); - KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new KeyedStateReaderOperator<>(new TimerReaderFunction(), Types.INT)); + KeyedStateInputFormat format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new TimerReaderFunction(), Types.INT)); KeyGroupRangeInputSplit split = format.createInputSplits(1)[0]; KeyedStateReaderFunction userFunction = new TimerReaderFunction(); @@ -162,6 +162,7 @@ private List readInputSplit(KeyGroupRangeInputSplit split, KeyedStateRe KeyedStateInputFormat format = new KeyedStateInputFormat<>( new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4), new MemoryStateBackend(), + new Configuration(), new KeyedStateReaderOperator<>(userFunction, Types.INT)); List data = new ArrayList<>(); From 045ff18faec85b19baaa9691e24a1eccc5460d9b Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 4 May 2020 20:17:38 +0200 Subject: [PATCH 023/773] [FLINK-16998][core] Add a changeflag to Row Partial commit for supporting a changeflag without backwards compatibility. This closes #12103. --- .../flink/api/common/typeinfo/Types.java | 2 +- .../flink/api/java/typeutils/RowTypeInfo.java | 15 ++ .../{NullMaskUtils.java => MaskUtils.java} | 37 ++-- .../java/typeutils/runtime/RowComparator.java | 66 +++--- .../java/typeutils/runtime/RowSerializer.java | 197 ++++++++++++------ .../main/java/org/apache/flink/types/Row.java | 173 ++++++++++----- .../java/org/apache/flink/types/RowKind.java | 3 + .../runtime/LegacyRowSerializerTest.java | 184 ++++++++++++++++ .../typeutils/runtime/RowComparatorTest.java | 34 +-- .../runtime/RowSerializerMigrationTest.java | 2 + .../typeutils/runtime/RowSerializerTest.java | 42 ++-- .../runtime/typeutils/PythonTypeUtils.java | 2 +- .../serializers/python/RowDataSerializer.java | 8 +- .../sources/RowArrowSourceFunctionTest.java | 2 +- .../flink/table/data/GenericRowData.java | 30 +++ .../data/conversion/RowRowConverter.java | 4 +- .../table/data/util/DataFormatConverters.java | 4 +- .../table/data/DataFormatConvertersTest.java | 3 +- .../data/DataStructureConvertersTest.java | 5 +- 19 files changed, 606 insertions(+), 207 deletions(-) rename flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/{NullMaskUtils.java => MaskUtils.java} (73%) create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/LegacyRowSerializerTest.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java index ede4396a468c1..34e0bef62e282 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/Types.java @@ -183,7 +183,7 @@ public class Types { *

A row is a fixed-length, null-aware composite type for storing multiple values in a * deterministic field order. Every field can be null regardless of the field's type. * The type of row fields cannot be automatically inferred; therefore, it is required to provide - * type information whenever a row is used. + * type information whenever a row is produced. * *

The schema of rows can have up to Integer.MAX_VALUE fields, however, all row instances * must strictly adhere to the schema defined by the type info. diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java index aec070cb17de4..e99e0bcbe68ee 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/RowTypeInfo.java @@ -286,6 +286,21 @@ public String toString() { return bld.toString(); } + /** + * Creates a serializer for the old {@link Row} format before Flink 1.11. + * + *

The serialization format has changed from 1.10 to 1.11 and added {@link Row#getKind()}. + */ + @Deprecated + public TypeSerializer createLegacySerializer(ExecutionConfig config) { + int len = getArity(); + TypeSerializer[] fieldSerializers = new TypeSerializer[len]; + for (int i = 0; i < len; i++) { + fieldSerializers[i] = types[i].createSerializer(config); + } + return new RowSerializer(fieldSerializers, true); + } + /** * Returns the field types of the row. The order matches the order of the field names. */ diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/NullMaskUtils.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/MaskUtils.java similarity index 73% rename from flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/NullMaskUtils.java rename to flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/MaskUtils.java index cfe562f5104e1..ea2830ec3a0da 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/NullMaskUtils.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/MaskUtils.java @@ -20,14 +20,19 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.types.Row; import java.io.IOException; +/** + * Utilities for reading and writing binary masks. + */ @Internal -public class NullMaskUtils { +public final class MaskUtils { + + @SuppressWarnings("UnusedAssignment") + public static void writeMask(boolean[] mask, DataOutputView target) throws IOException { + final int len = mask.length; - public static void writeNullMask(int len, Row value, DataOutputView target) throws IOException { int b = 0x00; int bytePos = 0; @@ -40,8 +45,8 @@ public static void writeNullMask(int len, Row value, DataOutputView target) thro numPos = Math.min(8, len - fieldPos); while (bytePos < numPos) { b = b << 1; - // set bit if field is null - if (value.getField(fieldPos + bytePos) == null) { + // set bit if element is true + if (mask[fieldPos + bytePos]) { b |= 0x01; } bytePos += 1; @@ -54,10 +59,9 @@ public static void writeNullMask(int len, Row value, DataOutputView target) thro } } - public static void readIntoNullMask( - int len, - DataInputView source, - boolean[] nullMask) throws IOException { + @SuppressWarnings("UnusedAssignment") + public static void readIntoMask(DataInputView source, boolean[] mask) throws IOException { + final int len = mask.length; int b = 0x00; int bytePos = 0; @@ -70,7 +74,7 @@ public static void readIntoNullMask( bytePos = 0; numPos = Math.min(8, len - fieldPos); while (bytePos < numPos) { - nullMask[fieldPos + bytePos] = (b & 0x80) > 0; + mask[fieldPos + bytePos] = (b & 0x80) > 0; b = b << 1; bytePos += 1; } @@ -78,11 +82,12 @@ public static void readIntoNullMask( } } - public static void readIntoAndCopyNullMask( - int len, - DataInputView source, - DataOutputView target, - boolean[] nullMask) throws IOException { + @SuppressWarnings("UnusedAssignment") + public static void readIntoAndCopyMask( + DataInputView source, + DataOutputView target, + boolean[] mask) throws IOException { + final int len = mask.length; int b = 0x00; int bytePos = 0; @@ -97,7 +102,7 @@ public static void readIntoAndCopyNullMask( bytePos = 0; numPos = Math.min(8, len - fieldPos); while (bytePos < numPos) { - nullMask[fieldPos + bytePos] = (b & 0x80) > 0; + mask[fieldPos + bytePos] = (b & 0x80) > 0; b = b << 1; bytePos += 1; } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowComparator.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowComparator.java index 135623bcf2f89..3f801001b11e4 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowComparator.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowComparator.java @@ -32,16 +32,21 @@ import java.util.Collections; import java.util.List; -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoNullMask; +import static org.apache.flink.api.java.typeutils.runtime.MaskUtils.readIntoMask; +import static org.apache.flink.api.java.typeutils.runtime.RowSerializer.ROW_KIND_OFFSET; import static org.apache.flink.util.Preconditions.checkArgument; /** - * Comparator for {@link Row} + * Comparator for {@link Row}. + * + *

Note: Since comparators are used only in DataSet API for batch use cases, this comparator assumes the + * latest serialization format and ignores {@link Row#getKind()} for simplicity of the implementation + * and efficiency. */ @Internal public class RowComparator extends CompositeTypeComparator { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; /** The number of fields of the Row */ private final int arity; /** key positions describe which fields are keys in what order */ @@ -56,9 +61,10 @@ public class RowComparator extends CompositeTypeComparator { private final int normalizableKeyPrefixLen; private final boolean invertNormKey; - // null masks for serialized comparison - private final boolean[] nullMask1; - private final boolean[] nullMask2; + // bitmask for serialized comparison + // see serializer for more information about the bitmask encoding + private final boolean[] mask1; + private final boolean[] mask2; // cache for the deserialized key field objects transient private final Object[] deserializedKeyFields1; @@ -144,8 +150,8 @@ private RowComparator( this.numLeadingNormalizableKeys = numLeadingNormalizableKeys; this.normalizableKeyPrefixLen = normalizableKeyPrefixLen; this.invertNormKey = invertNormKey; - this.nullMask1 = new boolean[arity]; - this.nullMask2 = new boolean[arity]; + this.mask1 = new boolean[ROW_KIND_OFFSET + arity]; + this.mask2 = new boolean[ROW_KIND_OFFSET + arity]; deserializedKeyFields1 = instantiateDeserializationFields(); deserializedKeyFields2 = instantiateDeserializationFields(); } @@ -251,43 +257,43 @@ public int compare(Row first, Row second) { @Override public int compareSerialized( - DataInputView firstSource, - DataInputView secondSource) throws IOException { - - int len = serializers.length; - int keyLen = keyPositions.length; + DataInputView firstSource, + DataInputView secondSource) throws IOException { + final int len = serializers.length; + final int keyLen = keyPositions.length; - readIntoNullMask(arity, firstSource, nullMask1); - readIntoNullMask(arity, secondSource, nullMask2); + // read bitmask + readIntoMask(firstSource, mask1); + readIntoMask(secondSource, mask2); - // deserialize - for (int i = 0; i < len; i++) { - TypeSerializer serializer = serializers[i]; + // deserialize fields + for (int fieldPos = 0; fieldPos < len; fieldPos++) { + final TypeSerializer serializer = serializers[fieldPos]; // deserialize field 1 - if (!nullMask1[i]) { - deserializedKeyFields1[i] = serializer.deserialize( - deserializedKeyFields1[i], + if (!mask1[ROW_KIND_OFFSET + fieldPos]) { + deserializedKeyFields1[fieldPos] = serializer.deserialize( + deserializedKeyFields1[fieldPos], firstSource); } // deserialize field 2 - if (!nullMask2[i]) { - deserializedKeyFields2[i] = serializer.deserialize( - deserializedKeyFields2[i], + if (!mask2[ROW_KIND_OFFSET + fieldPos]) { + deserializedKeyFields2[fieldPos] = serializer.deserialize( + deserializedKeyFields2[fieldPos], secondSource); } } // compare - for (int i = 0; i < keyLen; i++) { - int keyPos = keyPositions[i]; - TypeComparator comparator = comparators[i]; + for (int fieldPos = 0; fieldPos < keyLen; fieldPos++) { + final int keyPos = keyPositions[fieldPos]; + final TypeComparator comparator = comparators[fieldPos]; - boolean isNull1 = nullMask1[keyPos]; - boolean isNull2 = nullMask2[keyPos]; + final boolean isNull1 = mask1[ROW_KIND_OFFSET + keyPos]; + final boolean isNull2 = mask2[ROW_KIND_OFFSET + keyPos]; - int cmp = 0; + int cmp; // both values are null -> equality if (isNull1 && isNull2) { cmp = 0; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java index 505ce7bb915a8..a4924574dcde5 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java @@ -28,35 +28,61 @@ import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; import java.io.IOException; import java.io.ObjectInputStream; import java.util.Arrays; +import java.util.Objects; -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoAndCopyNullMask; -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoNullMask; -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.writeNullMask; +import static org.apache.flink.api.java.typeutils.runtime.MaskUtils.readIntoAndCopyMask; +import static org.apache.flink.api.java.typeutils.runtime.MaskUtils.readIntoMask; +import static org.apache.flink.api.java.typeutils.runtime.MaskUtils.writeMask; import static org.apache.flink.util.Preconditions.checkNotNull; /** * Serializer for {@link Row}. + * + *

It uses the following serialization format: + * 

+ *     |bitmask|field|field|....
+ *
+ * The bitmask serves as a header that consists of {@link #ROW_KIND_OFFSET} bits for encoding the + * {@link RowKind} and n bits for whether a field is null. For backwards compatibility, those bits + * can be ignored if serializer runs in legacy mode: + * 
+ *     bitmask with row kind:  |RK RK F1 F2 ... FN|
+ *     bitmask in legacy mode: |F1 F2 ... FN|
+ *
*/ @Internal public final class RowSerializer extends TypeSerializer { - private static final long serialVersionUID = 1L; + public static final int ROW_KIND_OFFSET = 2; + + private static final long serialVersionUID = 2L; + + private final boolean legacyModeEnabled; + + private final int legacyOffset; private final TypeSerializer[] fieldSerializers; private final int arity; - private transient boolean[] nullMask; + private transient boolean[] mask; - @SuppressWarnings("unchecked") public RowSerializer(TypeSerializer[] fieldSerializers) { + this(fieldSerializers, false); + } + + @SuppressWarnings("unchecked") + public RowSerializer(TypeSerializer[] fieldSerializers, boolean legacyModeEnabled) { + this.legacyModeEnabled = legacyModeEnabled; + this.legacyOffset = legacyModeEnabled ? 0 : ROW_KIND_OFFSET; this.fieldSerializers = (TypeSerializer[]) checkNotNull(fieldSerializers); this.arity = fieldSerializers.length; - this.nullMask = new boolean[fieldSerializers.length]; + this.mask = new boolean[legacyOffset + fieldSerializers.length]; } @Override @@ -70,7 +96,7 @@ public TypeSerializer duplicate() { for (int i = 0; i < fieldSerializers.length; i++) { duplicateFieldSerializers[i] = fieldSerializers[i].duplicate(); } - return new RowSerializer(duplicateFieldSerializers); + return new RowSerializer(duplicateFieldSerializers, legacyModeEnabled); } @Override @@ -86,7 +112,7 @@ public Row copy(Row from) { throw new RuntimeException("Row arity of from does not match serializers."); } - Row result = new Row(len); + Row result = new Row(from.getKind(), len); for (int i = 0; i < len; i++) { Object fromField = from.getField(i); if (fromField != null) { @@ -114,6 +140,8 @@ public Row copy(Row from, Row reuse) { "Row arity of reuse or from is incompatible with this RowSerializer."); } + reuse.setKind(from.getKind()); + for (int i = 0; i < len; i++) { Object fromField = from.getField(i); if (fromField != null) { @@ -145,39 +173,42 @@ public int getArity() { @Override public void serialize(Row record, DataOutputView target) throws IOException { - int len = fieldSerializers.length; + final int len = fieldSerializers.length; if (record.getArity() != len) { throw new RuntimeException("Row arity of from does not match serializers."); } - // write a null mask - writeNullMask(len, record, target); + // write bitmask + fillMask(len, record, mask, legacyModeEnabled, legacyOffset); + writeMask(mask, target); // serialize non-null fields - for (int i = 0; i < len; i++) { - Object o = record.getField(i); + for (int fieldPos = 0; fieldPos < len; fieldPos++) { + final Object o = record.getField(fieldPos); if (o != null) { - fieldSerializers[i].serialize(o, target); + fieldSerializers[fieldPos].serialize(o, target); } } } @Override public Row deserialize(DataInputView source) throws IOException { - int len = fieldSerializers.length; - - Row result = new Row(len); - - // read null mask - readIntoNullMask(len, source, nullMask); + final int len = fieldSerializers.length; + + // read bitmask + readIntoMask(source, mask); + final Row result; + if (legacyModeEnabled) { + result = new Row(len); + } else { + result = new Row(readKindFromMask(mask), len); + } - for (int i = 0; i < len; i++) { - if (nullMask[i]) { - result.setField(i, null); - } - else { - result.setField(i, fieldSerializers[i].deserialize(source)); + // deserialize fields + for (int fieldPos = 0; fieldPos < len; fieldPos++) { + if (!mask[legacyOffset + fieldPos]) { + result.setField(fieldPos, fieldSerializers[fieldPos].deserialize(source)); } } @@ -186,26 +217,29 @@ public Row deserialize(DataInputView source) throws IOException { @Override public Row deserialize(Row reuse, DataInputView source) throws IOException { - int len = fieldSerializers.length; + final int len = fieldSerializers.length; if (reuse.getArity() != len) { throw new RuntimeException("Row arity of from does not match serializers."); } - // read null mask - readIntoNullMask(len, source, nullMask); + // read bitmask + readIntoMask(source, mask); + if (!legacyModeEnabled) { + reuse.setKind(readKindFromMask(mask)); + } - for (int i = 0; i < len; i++) { - if (nullMask[i]) { - reuse.setField(i, null); - } - else { - Object reuseField = reuse.getField(i); + // deserialize fields + for (int fieldPos = 0; fieldPos < len; fieldPos++) { + if (mask[legacyOffset + fieldPos]) { + reuse.setField(fieldPos, null); + } else { + Object reuseField = reuse.getField(fieldPos); if (reuseField != null) { - reuse.setField(i, fieldSerializers[i].deserialize(reuseField, source)); + reuse.setField(fieldPos, fieldSerializers[fieldPos].deserialize(reuseField, source)); } else { - reuse.setField(i, fieldSerializers[i].deserialize(source)); + reuse.setField(fieldPos, fieldSerializers[fieldPos].deserialize(source)); } } } @@ -217,43 +251,68 @@ public Row deserialize(Row reuse, DataInputView source) throws IOException { public void copy(DataInputView source, DataOutputView target) throws IOException { int len = fieldSerializers.length; - // copy null mask - readIntoAndCopyNullMask(len, source, target, nullMask); + // copy bitmask + readIntoAndCopyMask(source, target, mask); - for (int i = 0; i < len; i++) { - if (!nullMask[i]) { - fieldSerializers[i].copy(source, target); + // copy non-null fields + for (int fieldPos = 0; fieldPos < len; fieldPos++) { + if (!mask[legacyOffset + fieldPos]) { + fieldSerializers[fieldPos].copy(source, target); } } } @Override - public boolean equals(Object obj) { - if (obj instanceof RowSerializer) { - RowSerializer other = (RowSerializer) obj; - if (this.fieldSerializers.length == other.fieldSerializers.length) { - for (int i = 0; i < this.fieldSerializers.length; i++) { - if (!this.fieldSerializers[i].equals(other.fieldSerializers[i])) { - return false; - } - } - return true; - } + public boolean equals(Object o) { + if (this == o) { + return true; } - - return false; + if (o == null || getClass() != o.getClass()) { + return false; + } + RowSerializer that = (RowSerializer) o; + return legacyModeEnabled == that.legacyModeEnabled && + Arrays.equals(fieldSerializers, that.fieldSerializers); } @Override public int hashCode() { - return Arrays.hashCode(fieldSerializers); + int result = Objects.hash(legacyModeEnabled); + result = 31 * result + Arrays.hashCode(fieldSerializers); + return result; } // -------------------------------------------------------------------------------------------- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); - this.nullMask = new boolean[fieldSerializers.length]; + this.mask = new boolean[legacyOffset + fieldSerializers.length]; + } + + // -------------------------------------------------------------------------------------------- + // Serialization utilities + // -------------------------------------------------------------------------------------------- + + private static void fillMask( + int fieldLength, + Row row, + boolean[] mask, + boolean legacyModeEnabled, + int legacyOffset) { + if (!legacyModeEnabled) { + final byte kind = row.getKind().toByteValue(); + mask[0] = (kind & 0x01) > 0; + mask[1] = (kind & 0x02) > 0; + } + + for (int fieldPos = 0; fieldPos < fieldLength; fieldPos++) { + mask[legacyOffset + fieldPos] = row.getField(fieldPos) == null; + } + } + + private static RowKind readKindFromMask(boolean[] mask) { + final byte kind = (byte) ((mask[0] ? 0x01 : 0x00) + (mask[1] ? 0x02 : 0x00)); + return RowKind.fromByteValue(kind); } // -------------------------------------------------------------------------------------------- @@ -282,7 +341,7 @@ public static final class RowSerializerConfigSnapshot extends CompositeTypeSeria public RowSerializerConfigSnapshot() { } - public RowSerializerConfigSnapshot(TypeSerializer[] fieldSerializers) { + public RowSerializerConfigSnapshot(TypeSerializer[] fieldSerializers) { super(fieldSerializers); } @@ -308,11 +367,15 @@ public TypeSerializerSchemaCompatibility resolveSchemaCompatibility(TypeSer /** * A {@link TypeSerializerSnapshot} for RowSerializer. */ + // TODO not fully functional yet due to FLINK-17520 public static final class RowSerializerSnapshot extends CompositeTypeSerializerSnapshot { - private static final int VERSION = 2; + private static final int VERSION = 3; + + private static final int VERSION_WITHOUT_ROW_KIND = 2; + + private boolean legacyModeEnabled = false; - @SuppressWarnings("WeakerAccess") public RowSerializerSnapshot() { super(RowSerializer.class); } @@ -326,6 +389,16 @@ protected int getCurrentOuterSnapshotVersion() { return VERSION; } + @Override + protected void readOuterSnapshot( + int readOuterSnapshotVersion, + DataInputView in, + ClassLoader userCodeClassLoader) { + if (readOuterSnapshotVersion == VERSION_WITHOUT_ROW_KIND) { + legacyModeEnabled = true; + } + } + @Override protected TypeSerializer[] getNestedSerializers(RowSerializer outerSerializer) { return outerSerializer.fieldSerializers; @@ -333,7 +406,7 @@ protected TypeSerializer[] getNestedSerializers(RowSerializer outerSerializer @Override protected RowSerializer createOuterSerializerWithNestedSerializers(TypeSerializer[] nestedSerializers) { - return new RowSerializer(nestedSerializers); + return new RowSerializer(nestedSerializers, legacyModeEnabled); } } } diff --git a/flink-core/src/main/java/org/apache/flink/types/Row.java b/flink-core/src/main/java/org/apache/flink/types/Row.java index aa15bf9e89d9f..29ec0585d6caf 100644 --- a/flink-core/src/main/java/org/apache/flink/types/Row.java +++ b/flink-core/src/main/java/org/apache/flink/types/Row.java @@ -18,68 +18,118 @@ package org.apache.flink.types; import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.util.Preconditions; import org.apache.flink.util.StringUtils; +import javax.annotation.Nullable; + import java.io.Serializable; import java.util.Arrays; /** - * A Row can have arbitrary number of fields and contain a set of fields, which may all be - * different types. The fields in Row can be null. Due to Row is not strongly typed, Flink's - * type extraction mechanism can't extract correct field types. So that users should manually - * tell Flink the type information via creating a {@link RowTypeInfo}. + * A row is a fixed-length, null-aware composite type for storing multiple values in a deterministic + * field order. Every field can be null regardless of the field's type. The type of row fields cannot + * be automatically inferred; therefore, it is required to provide type information whenever a row is + * produced. + * + *

The main purpose of rows is to bridge between Flink's Table and SQL ecosystem and other APIs. Therefore, + * a row does not only consist of a schema part (containing the fields) but also attaches a {@link RowKind} + * for encoding a change in a changelog. Thus, a row can be considered as an entry in a changelog. For example, + * in regular batch scenarios, a changelog would consist of a bounded stream of {@link RowKind#INSERT} rows. * - *

- * The fields in the Row can be accessed by position (zero-based) {@link #getField(int)}. And can - * set fields by {@link #setField(int, Object)}. - *

- * Row is in principle serializable. However, it may contain non-serializable fields, - * in which case serialization will fail. + *

The fields of a row can be accessed by position (zero-based) using {@link #getField(int)} and + * {@link #setField(int, Object)}. The row kind is kept separate from the fields and can be accessed + * by using {@link #getKind()} and {@link #setKind(RowKind)}. * + *

A row instance is in principle {@link Serializable}. However, it may contain non-serializable fields + * in which case serialization will fail if the row is not serialized with Flink's serialization stack. */ @PublicEvolving -public class Row implements Serializable{ +public final class Row implements Serializable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; + + /** The kind of change a row describes in a changelog. */ + private RowKind kind; /** The array to store actual values. */ private final Object[] fields; /** - * Create a new Row instance. - * @param arity The number of fields in the Row + * Create a new row instance. + * + *

By default, a row describes an {@link RowKind#INSERT} change. + * + * @param kind kind of change a row describes in a changelog + * @param arity The number of fields in the row. */ - public Row(int arity) { + public Row(RowKind kind, int arity) { + this.kind = Preconditions.checkNotNull(kind, "Row kind must not be null."); this.fields = new Object[arity]; } /** - * Get the number of fields in the Row. - * @return The number of fields in the Row. + * Create a new row instance. + * + *

By default, a row describes an {@link RowKind#INSERT} change. + * + * @param arity The number of fields in the row. + */ + public Row(int arity) { + this(RowKind.INSERT, arity); + } + + /** + * Returns the kind of change that this row describes in a changelog. + * + *

By default, a row describes an {@link RowKind#INSERT} change. + * + * @see RowKind + */ + public RowKind getKind() { + return kind; + } + + /** + * Sets the kind of change that this row describes in a changelog. + * + *

By default, a row describes an {@link RowKind#INSERT} change. + * + * @see RowKind + */ + public void setKind(RowKind kind) { + Preconditions.checkNotNull(kind, "Row kind must not be null."); + this.kind = kind; + } + + /** + * Returns the number of fields in the row. + * + *

Note: The row kind is kept separate from the fields and is not included in this number. + * + * @return The number of fields in the row. */ public int getArity() { return fields.length; } /** - * Gets the field at the specified position. + * Returns the field's content at the specified position. + * * @param pos The position of the field, 0-based. - * @return The field at the specified position. - * @throws IndexOutOfBoundsException Thrown, if the position is negative, or equal to, or larger than the number of fields. + * @return The field's content at the specified position. */ - public Object getField(int pos) { + public @Nullable Object getField(int pos) { return fields[pos]; } /** - * Sets the field at the specified position. + * Sets the field's content at the specified position. * * @param pos The position of the field, 0-based. * @param value The value to be assigned to the field at the specified position. - * @throws IndexOutOfBoundsException Thrown, if the position is negative, or equal to, or larger than the number of fields. */ - public void setField(int pos, Object value) { + public void setField(int pos, @Nullable Object value) { fields[pos] = value; } @@ -103,25 +153,29 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - Row row = (Row) o; - - return Arrays.deepEquals(fields, row.fields); + return kind == row.kind && + Arrays.deepEquals(fields, row.fields); } @Override public int hashCode() { - return Arrays.deepHashCode(fields); + int result = kind.toByteValue(); // for stable hash across JVM instances + result = 31 * result + Arrays.deepHashCode(fields); + return result; } + // -------------------------------------------------------------------------------------------- + // Utility methods + // -------------------------------------------------------------------------------------------- + /** - * Creates a new Row and assigns the given values to the Row's fields. + * Creates a new row and assigns the given values to the row's fields. * This is more convenient than using the constructor. * *

For example: - * * 

-	 *     Row.of("hello", true, 1L);}
+	 *     Row.of("hello", true, 1L);
 	 *
* instead of * 
@@ -131,6 +185,7 @@ public int hashCode() {
 	 *     row.setField(2, 1L);
 	 *
* + *

By default, a row describes an {@link RowKind#INSERT} change. */ public static Row of(Object... values) { Row row = new Row(values.length); @@ -141,27 +196,50 @@ public static Row of(Object... values) { } /** - * Creates a new Row which copied from another row. - * This method does not perform a deep copy. + * Creates a new row with given kind and assigns the given values to the row's fields. + * This is more convenient than using the constructor. + * + *

For example: + * 

+	 *     Row.ofKind(RowKind.INSERT, "hello", true, 1L);
+	 *
+ * instead of + * 
+	 *     Row row = new Row(3);
+	 *     row.setKind(RowKind.INSERT);
+	 *     row.setField(0, "hello");
+	 *     row.setField(1, true);
+	 *     row.setField(2, 1L);
+	 *
+ */ + public static Row ofKind(RowKind kind, Object... values) { + Row row = new Row(kind, values.length); + for (int i = 0; i < values.length; i++) { + row.setField(i, values[i]); + } + return row; + } + + /** + * Creates a new row which is copied from another row (including its {@link RowKind}). * - * @param row The row being copied. - * @return The cloned new Row + *

This method does not perform a deep copy. */ public static Row copy(Row row) { - final Row newRow = new Row(row.fields.length); + final Row newRow = new Row(row.kind, row.fields.length); System.arraycopy(row.fields, 0, newRow.fields, 0, row.fields.length); return newRow; } /** - * Creates a new Row with projected fields from another row. - * This method does not perform a deep copy. + * Creates a new row with projected fields and identical {@link RowKind} from another row. + * + *

This method does not perform a deep copy. * - * @param fields fields to be projected - * @return the new projected Row + * @param fields field indices to be projected */ public static Row project(Row row, int[] fields) { - final Row newRow = new Row(fields.length); + final Row newRow = new Row(row.kind, fields.length); for (int i = 0; i < fields.length; i++) { newRow.fields[i] = row.fields[fields[i]]; } @@ -169,12 +247,11 @@ public static Row project(Row row, int[] fields) { } /** - * Creates a new Row which fields are copied from the other rows. - * This method does not perform a deep copy. + * Creates a new row with fields that are copied from the other rows and appended to the resulting + * row in the given order. The {@link RowKind} of the first row determines the {@link RowKind} of + * the result. * - * @param first The first row being copied. - * @param remainings The other rows being copied. - * @return the joined new Row + *

This method does not perform a deep copy. */ public static Row join(Row first, Row... remainings) { int newLength = first.fields.length; @@ -182,7 +259,7 @@ public static Row join(Row first, Row... remainings) { newLength += remaining.fields.length; } - final Row joinedRow = new Row(newLength); + final Row joinedRow = new Row(first.kind, newLength); int index = 0; // copy the first row diff --git a/flink-core/src/main/java/org/apache/flink/types/RowKind.java b/flink-core/src/main/java/org/apache/flink/types/RowKind.java index a1acf7d291e57..eddc1b2365756 100644 --- a/flink-core/src/main/java/org/apache/flink/types/RowKind.java +++ b/flink-core/src/main/java/org/apache/flink/types/RowKind.java @@ -26,6 +26,9 @@ @PublicEvolving public enum RowKind { + // Note: Enums have no stable hash code across different JVMs, use toByteValue() for + // this purpose. + /** * Insertion operation. */ diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/LegacyRowSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/LegacyRowSerializerTest.java new file mode 100644 index 0000000000000..5c78b851a34b0 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/LegacyRowSerializerTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.java.typeutils.runtime; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeutils.SerializerTestInstance; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.tuple.Tuple3; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.api.java.typeutils.TypeExtractor; +import org.apache.flink.types.Row; + +import org.junit.Test; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Tests for the old serialization format of {@link Row} before Flink 1.11. + */ +public class LegacyRowSerializerTest { + + @Test + public void testRowSerializer() { + RowTypeInfo typeInfo = new RowTypeInfo( + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.STRING_TYPE_INFO); + Row row1 = new Row(2); + row1.setField(0, 1); + row1.setField(1, "a"); + + Row row2 = new Row(2); + row2.setField(0, 2); + row2.setField(1, null); + + TypeSerializer serializer = typeInfo.createLegacySerializer(new ExecutionConfig()); + RowSerializerTestInstance instance = new RowSerializerTestInstance(serializer, row1, row2); + instance.testAll(); + } + + @Test + public void testLargeRowSerializer() { + RowTypeInfo typeInfo = new RowTypeInfo( + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.STRING_TYPE_INFO); + + Row row = new Row(13); + row.setField(0, 2); + row.setField(1, null); + row.setField(3, null); + row.setField(4, null); + row.setField(5, null); + row.setField(6, null); + row.setField(7, null); + row.setField(8, null); + row.setField(9, null); + row.setField(10, null); + row.setField(11, null); + row.setField(12, "Test"); + + TypeSerializer serializer = typeInfo.createLegacySerializer(new ExecutionConfig()); + RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, row); + testInstance.testAll(); + } + + @Test + public void testRowSerializerWithComplexTypes() { + RowTypeInfo typeInfo = new RowTypeInfo( + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.DOUBLE_TYPE_INFO, + BasicTypeInfo.STRING_TYPE_INFO, + new TupleTypeInfo>( + BasicTypeInfo.INT_TYPE_INFO, + BasicTypeInfo.BOOLEAN_TYPE_INFO, + BasicTypeInfo.SHORT_TYPE_INFO), + TypeExtractor.createTypeInfo(MyPojo.class)); + + MyPojo testPojo1 = new MyPojo(); + testPojo1.name = null; + MyPojo testPojo2 = new MyPojo(); + testPojo2.name = "Test1"; + MyPojo testPojo3 = new MyPojo(); + testPojo3.name = "Test2"; + + Row[] data = new Row[]{ + createRow(null, null, null, null, null), + createRow(0, null, null, null, null), + createRow(0, 0.0, null, null, null), + createRow(0, 0.0, "a", null, null), + createRow(1, 0.0, "a", null, null), + createRow(1, 1.0, "a", null, null), + createRow(1, 1.0, "b", null, null), + createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null), + createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null), + createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null), + createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null), + createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1), + createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2), + createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3) + }; + + TypeSerializer serializer = typeInfo.createLegacySerializer(new ExecutionConfig()); + RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data); + testInstance.testAll(); + } + + // ---------------------------------------------------------------------------------------------- + + private static Row createRow(Object f0, Object f1, Object f2, Object f3, Object f4) { + Row row = new Row(5); + row.setField(0, f0); + row.setField(1, f1); + row.setField(2, f2); + row.setField(3, f3); + row.setField(4, f4); + return row; + } + + private class RowSerializerTestInstance extends SerializerTestInstance { + + RowSerializerTestInstance( + TypeSerializer serializer, + Row... testData) { + super(serializer, Row.class, -1, testData); + } + } + + public static class MyPojo implements Serializable, Comparable { + public String name = null; + + @Override + public int compareTo(MyPojo o) { + if (name == null && o.name == null) { + return 0; + } else if (name == null) { + return -1; + } else if (o.name == null) { + return 1; + } else { + return name.compareTo(o.name); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final MyPojo myPojo = (MyPojo) o; + return Objects.equals(name, myPojo.name); + } + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowComparatorTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowComparatorTest.java index ca54bd471539b..471c69277e316 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowComparatorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowComparatorTest.java @@ -27,6 +27,8 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + import org.junit.BeforeClass; import java.io.Serializable; @@ -51,20 +53,20 @@ public class RowComparatorTest extends ComparatorTestBase { private static MyPojo testPojo3 = new MyPojo(); private static final Row[] data = new Row[]{ - createRow(null, null, null, null, null), - createRow(0, null, null, null, null), - createRow(0, 0.0, null, null, null), - createRow(0, 0.0, "a", null, null), - createRow(1, 0.0, "a", null, null), - createRow(1, 1.0, "a", null, null), - createRow(1, 1.0, "b", null, null), - createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null), - createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3) + createRow(RowKind.INSERT, null, null, null, null, null), + createRow(RowKind.INSERT, 0, null, null, null, null), + createRow(RowKind.INSERT, 0, 0.0, null, null, null), + createRow(RowKind.INSERT, 0, 0.0, "a", null, null), + createRow(RowKind.INSERT, 1, 0.0, "a", null, null), + createRow(RowKind.INSERT, 1, 1.0, "a", null, null), + createRow(RowKind.INSERT, 1, 1.0, "b", null, null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null), + createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1), + createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2), + createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3) }; @BeforeClass @@ -110,8 +112,8 @@ protected boolean supportsNullKeys() { return true; } - private static Row createRow(Object f0, Object f1, Object f2, Object f3, Object f4) { - Row row = new Row(5); + private static Row createRow(RowKind kind, Object f0, Object f1, Object f2, Object f3, Object f4) { + Row row = new Row(kind, 5); row.setField(0, f0); row.setField(1, f1); row.setField(2, f2); diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerMigrationTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerMigrationTest.java index 7a7888aca61b4..1aacd4fe236a3 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerMigrationTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerMigrationTest.java @@ -27,6 +27,7 @@ import org.apache.flink.testutils.migration.MigrationVersion; import org.apache.flink.types.Row; +import org.junit.Ignore; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -35,6 +36,7 @@ /** * State migration test for {@link RowSerializer}. */ +@Ignore @RunWith(Parameterized.class) public class RowSerializerMigrationTest extends TypeSerializerSnapshotMigrationTestBase { diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java index f94943183839d..d11797496aa24 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/RowSerializerTest.java @@ -27,10 +27,12 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; import org.junit.Test; import java.io.Serializable; +import java.util.Objects; public class RowSerializerTest { @@ -40,10 +42,12 @@ public void testRowSerializer() { BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); Row row1 = new Row(2); + row1.setKind(RowKind.UPDATE_BEFORE); row1.setField(0, 1); row1.setField(1, "a"); Row row2 = new Row(2); + row2.setKind(RowKind.INSERT); row2.setField(0, 2); row2.setField(1, null); @@ -108,20 +112,20 @@ public void testRowSerializerWithComplexTypes() { testPojo3.name = "Test2"; Row[] data = new Row[]{ - createRow(null, null, null, null, null), - createRow(0, null, null, null, null), - createRow(0, 0.0, null, null, null), - createRow(0, 0.0, "a", null, null), - createRow(1, 0.0, "a", null, null), - createRow(1, 1.0, "a", null, null), - createRow(1, 1.0, "b", null, null), - createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null), - createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2), - createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3) + createRow(RowKind.INSERT, null, null, null, null, null), + createRow(RowKind.INSERT, 0, null, null, null, null), + createRow(RowKind.INSERT, 0, 0.0, null, null, null), + createRow(RowKind.INSERT, 0, 0.0, "a", null, null), + createRow(RowKind.INSERT, 1, 0.0, "a", null, null), + createRow(RowKind.INSERT, 1, 1.0, "a", null, null), + createRow(RowKind.INSERT, 1, 1.0, "b", null, null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null), + createRow(RowKind.UPDATE_AFTER, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null), + createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1), + createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2), + createRow(RowKind.DELETE, 1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3) }; TypeSerializer serializer = typeInfo.createSerializer(new ExecutionConfig()); @@ -131,8 +135,8 @@ public void testRowSerializerWithComplexTypes() { // ---------------------------------------------------------------------------------------------- - private static Row createRow(Object f0, Object f1, Object f2, Object f3, Object f4) { - Row row = new Row(5); + private static Row createRow(RowKind kind, Object f0, Object f1, Object f2, Object f3, Object f4) { + Row row = new Row(kind, 5); row.setField(0, f0); row.setField(1, f1); row.setField(2, f2); @@ -175,10 +179,8 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - - MyPojo myPojo = (MyPojo) o; - - return name != null ? name.equals(myPojo.name) : myPojo.name == null; + final MyPojo myPojo = (MyPojo) o; + return Objects.equals(name, myPojo.name); } } diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java index c919eb7a1d957..1ebcc0a8e9f0f 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java @@ -292,7 +292,7 @@ public TypeSerializer visit(RowType rowType) { .stream() .map(f -> f.getType().accept(this)) .toArray(TypeSerializer[]::new); - return new RowSerializer(fieldTypeSerializers); + return new RowSerializer(fieldTypeSerializers, true); } @Override diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/serializers/python/RowDataSerializer.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/serializers/python/RowDataSerializer.java index 68d5250830b87..1fe81154ed265 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/serializers/python/RowDataSerializer.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/serializers/python/RowDataSerializer.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.Arrays; -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoNullMask; +import static org.apache.flink.api.java.typeutils.runtime.MaskUtils.readIntoMask; /** * A {@link TypeSerializer} for {@link RowData}. It should be noted that the header will not be encoded. @@ -55,7 +55,7 @@ public RowDataSerializer(LogicalType[] types, TypeSerializer[] fieldSerializers) super(types, fieldSerializers); this.fieldTypes = types; this.fieldSerializers = fieldSerializers; - this.nullMask = new boolean[fieldTypes.length]; + this.nullMask = new boolean[fieldSerializers.length]; } @Override @@ -79,10 +79,8 @@ public void serialize(RowData row, DataOutputView target) throws IOException { @Override public RowData deserialize(DataInputView source) throws IOException { - int len = fieldSerializers.length; - // read null mask - readIntoNullMask(len, source, nullMask); + readIntoMask(source, nullMask); GenericRowData row = new GenericRowData(fieldSerializers.length); for (int i = 0; i < row.getArity(); i++) { diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/sources/RowArrowSourceFunctionTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/sources/RowArrowSourceFunctionTest.java index 46f2adaed715b..9b59650743d20 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/sources/RowArrowSourceFunctionTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/arrow/sources/RowArrowSourceFunctionTest.java @@ -52,7 +52,7 @@ public class RowArrowSourceFunctionTest extends ArrowSourceFunctionTestBase public RowArrowSourceFunctionTest() { super(VectorSchemaRoot.create(ArrowUtils.toArrowSchema(rowType), allocator), - new RowSerializer(new TypeSerializer[]{StringSerializer.INSTANCE}), + new RowSerializer(new TypeSerializer[]{StringSerializer.INSTANCE}, true), Comparator.comparing(o -> (String) (o.getField(0)))); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java index 411e65e1fd966..4687785df1e35 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java @@ -53,6 +53,21 @@ public final class GenericRowData implements RowData { /** The kind of change that a row describes in a changelog. */ private RowKind kind; + /** + * Creates an instance of {@link GenericRowData} with given kind and number of fields. + * + *

Initially, all fields are set to null. + * + *

Note: All fields of the row must be internal data structures. + * + * @param kind kind of change that this row describes in a changelog + * @param arity number of fields + */ + public GenericRowData(RowKind kind, int arity) { + this.fields = new Object[arity]; + this.kind = kind; + } + /** * Creates an instance of {@link GenericRowData} with given number of fields. * @@ -244,4 +259,19 @@ public static GenericRowData of(Object... values) { return row; } + + /** + * Creates an instance of {@link GenericRowData} with given kind and field values. + * + *

Note: All fields of the row must be internal data structures. + */ + public static GenericRowData ofKind(RowKind kind, Object... values) { + GenericRowData row = new GenericRowData(kind, values.length); + + for (int i = 0; i < values.length; ++i) { + row.setField(i, values[i]); + } + + return row; + } } diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java index 468efcd755277..c899ff39f0f15 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java @@ -57,7 +57,7 @@ public void open(ClassLoader classLoader) { @Override public RowData toInternal(Row external) { final int length = fieldConverters.length; - final GenericRowData genericRow = new GenericRowData(length); + final GenericRowData genericRow = new GenericRowData(external.getKind(), length); for (int pos = 0; pos < length; pos++) { final Object value = external.getField(pos); genericRow.setField(pos, fieldConverters[pos].toInternalOrNull(value)); @@ -68,7 +68,7 @@ public RowData toInternal(Row external) { @Override public Row toExternal(RowData internal) { final int length = fieldConverters.length; - final Row row = new Row(length); + final Row row = new Row(internal.getRowKind(), length); for (int pos = 0; pos < length; pos++) { final Object value = fieldGetters[pos].getFieldOrNull(internal); row.setField(pos, fieldConverters[pos].toExternalOrNull(value)); diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java index 1f1089c684e8a..347a517850b72 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java @@ -1411,7 +1411,7 @@ public RowConverter(DataType[] fieldTypes) { @Override RowData toInternalImpl(Row value) { - GenericRowData genericRow = new GenericRowData(converters.length); + GenericRowData genericRow = new GenericRowData(value.getKind(), converters.length); for (int i = 0; i < converters.length; i++) { genericRow.setField(i, converters[i].toInternal(value.getField(i))); } @@ -1420,7 +1420,7 @@ RowData toInternalImpl(Row value) { @Override Row toExternalImpl(RowData value) { - Row row = new Row(converters.length); + Row row = new Row(value.getRowKind(), converters.length); for (int i = 0; i < converters.length; i++) { row.setField(i, converters[i].toExternal(value, i)); } diff --git a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java index 4ca0f1cdb4e65..82e33d1f1b76e 100644 --- a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java +++ b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataFormatConvertersTest.java @@ -48,6 +48,7 @@ import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.utils.TypeConversions; import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; import org.junit.Assert; import org.junit.Test; @@ -169,7 +170,7 @@ public void testTypes() { test(simpleTypes[i], simpleValues[i]); } test(new RowTypeInfo(simpleTypes), new Row(simpleTypes.length)); - test(new RowTypeInfo(simpleTypes), Row.of(simpleValues)); + test(new RowTypeInfo(simpleTypes), Row.ofKind(RowKind.DELETE, simpleValues)); test(new RowDataTypeInfo(new VarCharType(VarCharType.MAX_LENGTH), new IntType()), GenericRowData.of(StringData.fromString("hehe"), 111)); test(new RowDataTypeInfo(new VarCharType(VarCharType.MAX_LENGTH), new IntType()), GenericRowData.of(null, null)); diff --git a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataStructureConvertersTest.java b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataStructureConvertersTest.java index 8af45a4e3a2cc..a74517cb170f3 100644 --- a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataStructureConvertersTest.java +++ b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DataStructureConvertersTest.java @@ -27,6 +27,7 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.utils.DataTypeFactoryMock; import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; import org.apache.flink.util.InstantiationUtil; import org.junit.Rule; @@ -235,8 +236,8 @@ public static List testData() { ROW( FIELD("b_1", DOUBLE()), FIELD("b_2", BOOLEAN()))))) - .convertedTo(Row.class, Row.of(12, Row.of(2.0, null))) - .convertedTo(RowData.class, GenericRowData.of(12, GenericRowData.of(2.0, null))), + .convertedTo(Row.class, Row.ofKind(RowKind.DELETE, 12, Row.of(2.0, null))) + .convertedTo(RowData.class, GenericRowData.ofKind(RowKind.DELETE, 12, GenericRowData.of(2.0, null))), TestSpec .forDataType( From 6fa0c92567a9d81639481063e422dde19a26e35a Mon Sep 17 00:00:00 2001 From: Zhu Zhu Date: Mon, 18 May 2020 18:30:23 +0800 Subject: [PATCH 024/773] =?UTF-8?q?[FLINK-15813][runtime]=20Set=20default?= =?UTF-8?q?=20value=20of=20config=20=E2=80=9Cjobmanager.execution.failover?= =?UTF-8?q?-strategy=E2=80=9D=20to=20=E2=80=9Cregion=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/_includes/generated/all_jobmanager_section.html | 2 +- .../generated/expert_fault_tolerance_section.html | 2 +- docs/_includes/generated/job_manager_configuration.html | 2 +- .../org/apache/flink/configuration/JobManagerOptions.java | 4 ++-- .../executiongraph/failover/FailoverStrategyLoader.java | 4 +++- .../failover/flip1/FailoverStrategyFactoryLoader.java | 8 +------- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/_includes/generated/all_jobmanager_section.html b/docs/_includes/generated/all_jobmanager_section.html index 6ef552c064896..e280984de5210 100644 --- a/docs/_includes/generated/all_jobmanager_section.html +++ b/docs/_includes/generated/all_jobmanager_section.html @@ -22,7 +22,7 @@

jobmanager.execution.failover-strategy
- region + "region" String This option specifies how the job computation recovers from task failures. Accepted values are:
  • 'full': Restarts all tasks to recover the job.
  • 'region': Restarts all tasks that could be affected by the task failure. More details can be found here.
diff --git a/docs/_includes/generated/expert_fault_tolerance_section.html b/docs/_includes/generated/expert_fault_tolerance_section.html index fb377b8ccecb2..8e3d6fcf6a6c2 100644 --- a/docs/_includes/generated/expert_fault_tolerance_section.html +++ b/docs/_includes/generated/expert_fault_tolerance_section.html @@ -58,7 +58,7 @@
jobmanager.execution.failover-strategy
- region + "region" String This option specifies how the job computation recovers from task failures. Accepted values are:
  • 'full': Restarts all tasks to recover the job.
  • 'region': Restarts all tasks that could be affected by the task failure. More details can be found here.
diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html index 52cfa13baccd3..87341aa45abe0 100644 --- a/docs/_includes/generated/job_manager_configuration.html +++ b/docs/_includes/generated/job_manager_configuration.html @@ -28,7 +28,7 @@
jobmanager.execution.failover-strategy
- region + "region" String This option specifies how the job computation recovers from task failures. Accepted values are:
  • 'full': Restarts all tasks to recover the job.
  • 'region': Restarts all tasks that could be affected by the task failure. More details can be found here.
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java index 173cdd2ea8f46..dad1497c383cc 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java @@ -234,10 +234,10 @@ public class JobManagerOptions { * This option specifies the failover strategy, i.e. how the job computation recovers from task failures. */ @Documentation.Section({Documentation.Sections.ALL_JOB_MANAGER, Documentation.Sections.EXPERT_FAULT_TOLERANCE}) - @Documentation.OverrideDefault("region") public static final ConfigOption EXECUTION_FAILOVER_STRATEGY = key("jobmanager.execution.failover-strategy") - .defaultValue("full") + .stringType() + .defaultValue("region") .withDescription(Description.builder() .text("This option specifies how the job computation recovers from task failures. " + "Accepted values are:") diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java index 6553c8a3f67e6..241cababc59fc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java @@ -41,7 +41,9 @@ public class FailoverStrategyLoader { * Loads a FailoverStrategy Factory from the given configuration. */ public static FailoverStrategy.Factory loadFailoverStrategy(Configuration config, @Nullable Logger logger) { - final String strategyParam = config.getString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY); + final String strategyParam = config.getString( + JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, + FULL_RESTART_STRATEGY_NAME); if (StringUtils.isNullOrWhitespaceOnly(strategyParam)) { if (logger != null) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailoverStrategyFactoryLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailoverStrategyFactoryLoader.java index 95dc7f64f3faf..68d614b4836d9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailoverStrategyFactoryLoader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailoverStrategyFactoryLoader.java @@ -47,13 +47,7 @@ private FailoverStrategyFactoryLoader() { public static FailoverStrategy.Factory loadFailoverStrategyFactory(final Configuration config) { checkNotNull(config); - // the default NG failover strategy is the region failover strategy. - // TODO: Remove the overridden default value when removing legacy scheduler - // and change the default value of JobManagerOptions.EXECUTION_FAILOVER_STRATEGY - // to be "region" - final String strategyParam = config.getString( - JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, - PIPELINED_REGION_RESTART_STRATEGY_NAME); + final String strategyParam = config.getString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY); switch (strategyParam.toLowerCase()) { case FULL_RESTART_STRATEGY_NAME: From 01852d7655898e14eec3452967862b924eac2b2f Mon Sep 17 00:00:00 2001 From: Gao Yun Date: Sun, 10 May 2020 22:09:26 +0800 Subject: [PATCH 025/773] [FLINK-17594][filesystem] Support Hadoop path-based part-file writer. --- flink-formats/flink-hadoop-bulk/pom.xml | 105 +++++++ .../DefaultHadoopFileCommitterFactory.java | 37 +++ .../hadoop/bulk/HadoopFileCommitter.java | 62 ++++ .../bulk/HadoopFileCommitterFactory.java | 45 +++ .../bulk/HadoopPathBasedBulkWriter.java | 71 +++++ .../bulk/HadoopPathBasedPartFileWriter.java | 270 +++++++++++++++++ .../committer/HadoopRenameFileCommitter.java | 107 +++++++ .../HadoopPathBasedBulkFormatBuilder.java | 147 ++++++++++ .../filesystem/SerializableConfiguration.java | 55 ++++ .../HadoopPathBasedPartFileWriterTest.java | 191 ++++++++++++ .../HadoopRenameFileCommitterTest.java | 275 ++++++++++++++++++ .../TestStreamingFileSinkFactory.java | 32 ++ .../src/test/resources/log4j2-test.properties | 28 ++ flink-formats/pom.xml | 1 + .../filesystem/AbstractPartFileWriter.java | 2 +- .../sink/filesystem/BucketWriter.java | 2 +- .../sink/filesystem/InProgressFileWriter.java | 2 +- .../sink/filesystem/StreamingFileSink.java | 22 +- .../sink/filesystem/WriterProperties.java | 2 +- 19 files changed, 1435 insertions(+), 21 deletions(-) create mode 100644 flink-formats/flink-hadoop-bulk/pom.xml create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/DefaultHadoopFileCommitterFactory.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitter.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitterFactory.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedBulkWriter.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriter.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitter.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/HadoopPathBasedBulkFormatBuilder.java create mode 100644 flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/SerializableConfiguration.java create mode 100644 flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java create mode 100644 flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitterTest.java create mode 100644 flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestStreamingFileSinkFactory.java create mode 100644 flink-formats/flink-hadoop-bulk/src/test/resources/log4j2-test.properties diff --git a/flink-formats/flink-hadoop-bulk/pom.xml b/flink-formats/flink-hadoop-bulk/pom.xml new file mode 100644 index 0000000000000..371c15a431736 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/pom.xml @@ -0,0 +1,105 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-formats + 1.11-SNAPSHOT + .. + + + flink-hadoop-bulk_${scala.binary.version} + flink-hadoop-bulk + + jar + + + + + + + org.apache.flink + flink-core + ${project.version} + provided + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + provided + + + + + + org.apache.hadoop + hadoop-common + provided + + + + org.apache.hadoop + hadoop-hdfs + provided + + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + + + + org.apache.flink + flink-test-utils_${scala.binary.version} + ${project.version} + test + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + dependency-convergence + + enforce + + + true + + + + + + + diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/DefaultHadoopFileCommitterFactory.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/DefaultHadoopFileCommitterFactory.java new file mode 100644 index 0000000000000..01ac88c7269d9 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/DefaultHadoopFileCommitterFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk; + +import org.apache.flink.formats.hadoop.bulk.committer.HadoopRenameFileCommitter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +/** + * The default hadoop file committer factory which always use {@link HadoopRenameFileCommitter}. + */ +public class DefaultHadoopFileCommitterFactory implements HadoopFileCommitterFactory { + + private static final long serialVersionUID = 1L; + + @Override + public HadoopFileCommitter create(Configuration configuration, Path targetFilePath) { + return new HadoopRenameFileCommitter(configuration, targetFilePath); + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitter.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitter.java new file mode 100644 index 0000000000000..7ae0d5679e7e0 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitter.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk; + +import org.apache.flink.annotation.Internal; + +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +/** + * The committer publishes an intermediate Hadoop file to the target path after + * it finishes writing. + */ +@Internal +public interface HadoopFileCommitter { + + /** + * Gets the target path to commit to. + * + * @return The target path to commit to. + */ + Path getTargetFilePath(); + + /** + * Gets the path of the intermediate file to commit. + * + * @return The path of the intermediate file to commit. + */ + Path getInProgressFilePath(); + + /** + * Prepares the intermediates file for committing. + */ + void preCommit() throws IOException; + + /** + * Commits the in-progress file to the target path. + */ + void commit() throws IOException; + + /** + * Re-commits the in-progress file to the target path after fail-over. + */ + void commitAfterRecovery() throws IOException; +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitterFactory.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitterFactory.java new file mode 100644 index 0000000000000..ae0495a5c9483 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopFileCommitterFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk; + +import org.apache.flink.annotation.Internal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.io.Serializable; + +/** + * The factory to create the {@link HadoopFileCommitter}. + */ +@Internal +public interface HadoopFileCommitterFactory extends Serializable { + + /** + * Creates the corresponding Hadoop file committer according to the Hadoop + * configuration and the target path. + * + * @param configuration The hadoop configuration. + * @param targetFilePath The target path to commit. + * @return The corresponding Hadoop file committer. + */ + HadoopFileCommitter create(Configuration configuration, Path targetFilePath) throws IOException; + +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedBulkWriter.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedBulkWriter.java new file mode 100644 index 0000000000000..7033730704ee7 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedBulkWriter.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.BulkWriter; + +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.io.Serializable; + +/** + * Specialized {@link BulkWriter} which is expected to write to specified + * {@link Path}. + */ +@Internal +public interface HadoopPathBasedBulkWriter extends BulkWriter { + + /** + * Gets the size written by the current writer. + * + * @return The size written by the current writer. + */ + long getSize() throws IOException; + + /** + * Disposes the writer on failures. Unlike output-stream-based writers which + * could handled uniformly by closing the underlying output stream, the path- + * based writers need to be disposed explicitly. + */ + void dispose(); + + // ------------------------------------------------------------------------ + + /** + * A factory that creates a {@link HadoopPathBasedBulkWriter}. + * + * @param The type of record to write. + */ + @FunctionalInterface + interface Factory extends Serializable { + + /** + * Creates a path-based writer that writes to the inProgressPath first + * and commits to targetPath finally. + * + * @param targetFilePath The final path to commit to. + * @param inProgressFilePath The intermediate path to write to before committing. + * @return The created writer. + */ + HadoopPathBasedBulkWriter create(Path targetFilePath, Path inProgressFilePath) throws IOException; + + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriter.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriter.java new file mode 100644 index 0000000000000..2703cfe534ac6 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriter.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.streaming.api.functions.sink.filesystem.AbstractPartFileWriter; +import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter; +import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter; +import org.apache.flink.streaming.api.functions.sink.filesystem.WriterProperties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * The part-file writer that writes to the specified hadoop path. + */ +public class HadoopPathBasedPartFileWriter extends AbstractPartFileWriter { + + private final HadoopPathBasedBulkWriter writer; + + private final HadoopFileCommitter fileCommitter; + + public HadoopPathBasedPartFileWriter( + final BucketID bucketID, + HadoopPathBasedBulkWriter writer, + HadoopFileCommitter fileCommitter, + long createTime) { + + super(bucketID, createTime); + + this.writer = writer; + this.fileCommitter = fileCommitter; + } + + @Override + public void write(IN element, long currentTime) throws IOException { + writer.addElement(element); + markWrite(currentTime); + } + + @Override + public InProgressFileRecoverable persist() { + throw new UnsupportedOperationException("The path based writers do not support persisting"); + } + + @Override + public PendingFileRecoverable closeForCommit() throws IOException { + writer.flush(); + writer.finish(); + fileCommitter.preCommit(); + return new HadoopPathBasedPendingFile(fileCommitter).getRecoverable(); + } + + @Override + public void dispose() { + writer.dispose(); + } + + @Override + public long getSize() throws IOException { + return writer.getSize(); + } + + static class HadoopPathBasedPendingFile implements BucketWriter.PendingFile { + private final HadoopFileCommitter fileCommitter; + + public HadoopPathBasedPendingFile(HadoopFileCommitter fileCommitter) { + this.fileCommitter = fileCommitter; + } + + @Override + public void commit() throws IOException { + fileCommitter.commit(); + } + + @Override + public void commitAfterRecovery() throws IOException { + fileCommitter.commitAfterRecovery(); + } + + public PendingFileRecoverable getRecoverable() { + return new HadoopPathBasedPendingFileRecoverable( + fileCommitter.getTargetFilePath()); + } + } + + @VisibleForTesting + static class HadoopPathBasedPendingFileRecoverable implements PendingFileRecoverable { + private final Path path; + + public HadoopPathBasedPendingFileRecoverable(Path path) { + this.path = path; + } + + public Path getPath() { + return path; + } + } + + @VisibleForTesting + static class HadoopPathBasedPendingFileRecoverableSerializer + implements SimpleVersionedSerializer { + + static final HadoopPathBasedPendingFileRecoverableSerializer INSTANCE = + new HadoopPathBasedPendingFileRecoverableSerializer(); + + private static final Charset CHARSET = StandardCharsets.UTF_8; + + private static final int MAGIC_NUMBER = 0x2c853c90; + + @Override + public int getVersion() { + return 1; + } + + @Override + public byte[] serialize(PendingFileRecoverable pendingFileRecoverable) { + if (!(pendingFileRecoverable instanceof HadoopPathBasedPartFileWriter.HadoopPathBasedPendingFileRecoverable)) { + throw new UnsupportedOperationException("Only HadoopPathBasedPendingFileRecoverable is supported."); + } + + Path path = ((HadoopPathBasedPendingFileRecoverable) pendingFileRecoverable).getPath(); + byte[] pathBytes = path.toUri().toString().getBytes(CHARSET); + + byte[] targetBytes = new byte[8 + pathBytes.length]; + ByteBuffer bb = ByteBuffer.wrap(targetBytes).order(ByteOrder.LITTLE_ENDIAN); + bb.putInt(MAGIC_NUMBER); + bb.putInt(pathBytes.length); + bb.put(pathBytes); + + return targetBytes; + } + + @Override + public HadoopPathBasedPendingFileRecoverable deserialize(int version, byte[] serialized) throws IOException { + switch (version) { + case 1: + return deserializeV1(serialized); + default: + throw new IOException("Unrecognized version or corrupt state: " + version); + } + } + + private HadoopPathBasedPendingFileRecoverable deserializeV1(byte[] serialized) throws IOException { + final ByteBuffer bb = ByteBuffer.wrap(serialized).order(ByteOrder.LITTLE_ENDIAN); + + if (bb.getInt() != MAGIC_NUMBER) { + throw new IOException("Corrupt data: Unexpected magic number."); + } + + byte[] pathBytes = new byte[bb.getInt()]; + bb.get(pathBytes); + String targetPath = new String(pathBytes, CHARSET); + + return new HadoopPathBasedPendingFileRecoverable(new Path(targetPath)); + } + } + + private static class UnsupportedInProgressFileRecoverableSerializable + implements SimpleVersionedSerializer { + + static final UnsupportedInProgressFileRecoverableSerializable INSTANCE = + new UnsupportedInProgressFileRecoverableSerializable(); + + @Override + public int getVersion() { + throw new UnsupportedOperationException("Persists the path-based part file write is not supported"); + } + + @Override + public byte[] serialize(InProgressFileRecoverable obj) { + throw new UnsupportedOperationException("Persists the path-based part file write is not supported"); + } + + @Override + public InProgressFileRecoverable deserialize(int version, byte[] serialized) { + throw new UnsupportedOperationException("Persists the path-based part file write is not supported"); + } + } + + /** + * Factory to create {@link HadoopPathBasedPartFileWriter}. + */ + public static class HadoopPathBasedBucketWriter implements BucketWriter { + private final Configuration configuration; + + private final HadoopPathBasedBulkWriter.Factory bulkWriterFactory; + + private final HadoopFileCommitterFactory fileCommitterFactory; + + public HadoopPathBasedBucketWriter( + Configuration configuration, + HadoopPathBasedBulkWriter.Factory bulkWriterFactory, + HadoopFileCommitterFactory fileCommitterFactory) { + + this.configuration = configuration; + this.bulkWriterFactory = bulkWriterFactory; + this.fileCommitterFactory = fileCommitterFactory; + } + + @Override + public HadoopPathBasedPartFileWriter openNewInProgressFile( + BucketID bucketID, + org.apache.flink.core.fs.Path flinkPath, + long creationTime) throws IOException { + + Path path = new Path(flinkPath.toUri()); + HadoopFileCommitter fileCommitter = fileCommitterFactory.create(configuration, path); + + Path inProgressFilePath = fileCommitter.getInProgressFilePath(); + HadoopPathBasedBulkWriter writer = bulkWriterFactory.create(path, inProgressFilePath); + return new HadoopPathBasedPartFileWriter<>(bucketID, writer, fileCommitter, creationTime); + } + + @Override + public PendingFile recoverPendingFile(PendingFileRecoverable pendingFileRecoverable) throws IOException { + if (!(pendingFileRecoverable instanceof HadoopPathBasedPartFileWriter.HadoopPathBasedPendingFileRecoverable)) { + throw new UnsupportedOperationException("Only HadoopPathBasedPendingFileRecoverable is supported."); + } + + Path path = ((HadoopPathBasedPendingFileRecoverable) pendingFileRecoverable).getPath(); + return new HadoopPathBasedPendingFile(fileCommitterFactory.create(configuration, path)); + } + + @Override + public WriterProperties getProperties() { + return new WriterProperties( + UnsupportedInProgressFileRecoverableSerializable.INSTANCE, + HadoopPathBasedPendingFileRecoverableSerializer.INSTANCE, + false); + } + + @Override + public InProgressFileWriter resumeInProgressFileFrom( + BucketID bucketID, + InProgressFileRecoverable inProgressFileSnapshot, + long creationTime) { + + throw new UnsupportedOperationException("Resume is not supported"); + } + + @Override + public boolean cleanupInProgressFileRecoverable(InProgressFileRecoverable inProgressFileRecoverable) { + return false; + } + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitter.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitter.java new file mode 100644 index 0000000000000..df4266bab132d --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitter.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk.committer; + +import org.apache.flink.formats.hadoop.bulk.HadoopFileCommitter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkArgument; + +/** + * The Hadoop file committer that directly rename the in-progress file + * to the target file. For FileSystem like S3, renaming may lead to + * additional copies. + */ +public class HadoopRenameFileCommitter implements HadoopFileCommitter { + + private final Configuration configuration; + + private final Path targetFilePath; + + private final Path inProgressFilePath; + + public HadoopRenameFileCommitter(Configuration configuration, Path targetFilePath) { + this.configuration = configuration; + this.targetFilePath = targetFilePath; + this.inProgressFilePath = generateInProgressFilePath(); + } + + @Override + public Path getTargetFilePath() { + return targetFilePath; + } + + @Override + public Path getInProgressFilePath() { + return inProgressFilePath; + } + + @Override + public void preCommit() { + // Do nothing. + } + + @Override + public void commit() throws IOException { + rename(true); + } + + @Override + public void commitAfterRecovery() throws IOException { + rename(false); + } + + private void rename(boolean assertFileExists) throws IOException { + FileSystem fileSystem = FileSystem.get(targetFilePath.toUri(), configuration); + + if (!fileSystem.exists(inProgressFilePath)) { + if (assertFileExists) { + throw new IOException(String.format("In progress file(%s) not exists.", inProgressFilePath)); + } else { + + // By pass the re-commit if source file not exists. + // TODO: in the future we may also need to check if the target file exists. + return; + } + } + + try { + // If file exists, it will be overwritten. + fileSystem.rename(inProgressFilePath, targetFilePath); + } catch (IOException e) { + throw new IOException( + String.format("Could not commit file from %s to %s", inProgressFilePath, targetFilePath), + e); + } + } + + private Path generateInProgressFilePath() { + checkArgument(targetFilePath.isAbsolute(), "Target file must be absolute"); + + Path parent = targetFilePath.getParent(); + String name = targetFilePath.getName(); + + return new Path(parent, "." + name + ".inprogress"); + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/HadoopPathBasedBulkFormatBuilder.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/HadoopPathBasedBulkFormatBuilder.java new file mode 100644 index 0000000000000..df51ff495f570 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/HadoopPathBasedBulkFormatBuilder.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.fs.Path; +import org.apache.flink.formats.hadoop.bulk.DefaultHadoopFileCommitterFactory; +import org.apache.flink.formats.hadoop.bulk.HadoopFileCommitterFactory; +import org.apache.flink.formats.hadoop.bulk.HadoopPathBasedBulkWriter; +import org.apache.flink.formats.hadoop.bulk.HadoopPathBasedPartFileWriter; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.CheckpointRollingPolicy; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy; +import org.apache.flink.util.Preconditions; + +import org.apache.hadoop.conf.Configuration; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** + * Buckets builder to create buckets that use {@link HadoopPathBasedPartFileWriter}. + */ +public class HadoopPathBasedBulkFormatBuilder> + extends StreamingFileSink.BucketsBuilder { + + private static final long serialVersionUID = 1L; + + private final Path basePath; + + private HadoopPathBasedBulkWriter.Factory writerFactory; + + private HadoopFileCommitterFactory fileCommitterFactory; + + private SerializableConfiguration serializableConfiguration; + + private BucketAssigner bucketAssigner; + + private CheckpointRollingPolicy rollingPolicy; + + @Nullable + private BucketLifeCycleListener bucketLifeCycleListener; + + private BucketFactory bucketFactory; + + private OutputFileConfig outputFileConfig; + + public HadoopPathBasedBulkFormatBuilder( + org.apache.hadoop.fs.Path basePath, + HadoopPathBasedBulkWriter.Factory writerFactory, + Configuration configuration, + BucketAssigner assigner) { + + this( + basePath, + writerFactory, + new DefaultHadoopFileCommitterFactory(), + configuration, + assigner, + OnCheckpointRollingPolicy.build(), + new DefaultBucketFactoryImpl<>(), + OutputFileConfig.builder().build()); + } + + public HadoopPathBasedBulkFormatBuilder( + org.apache.hadoop.fs.Path basePath, + HadoopPathBasedBulkWriter.Factory writerFactory, + HadoopFileCommitterFactory fileCommitterFactory, + Configuration configuration, + BucketAssigner assigner, + CheckpointRollingPolicy policy, + BucketFactory bucketFactory, + OutputFileConfig outputFileConfig) { + + this.basePath = new Path(Preconditions.checkNotNull(basePath).toString()); + this.writerFactory = writerFactory; + this.fileCommitterFactory = fileCommitterFactory; + this.serializableConfiguration = new SerializableConfiguration(configuration); + this.bucketAssigner = Preconditions.checkNotNull(assigner); + this.rollingPolicy = Preconditions.checkNotNull(policy); + this.bucketFactory = Preconditions.checkNotNull(bucketFactory); + this.outputFileConfig = Preconditions.checkNotNull(outputFileConfig); + } + + public T withBucketAssigner(BucketAssigner assigner) { + this.bucketAssigner = Preconditions.checkNotNull(assigner); + return self(); + } + + public T withRollingPolicy(CheckpointRollingPolicy rollingPolicy) { + this.rollingPolicy = Preconditions.checkNotNull(rollingPolicy); + return self(); + } + + @Internal + public T withBucketLifeCycleListener(final BucketLifeCycleListener listener) { + this.bucketLifeCycleListener = Preconditions.checkNotNull(listener); + return self(); + } + + public T withBucketFactory(BucketFactory factory) { + this.bucketFactory = Preconditions.checkNotNull(factory); + return self(); + } + + public T withOutputFileConfig(OutputFileConfig outputFileConfig) { + this.outputFileConfig = outputFileConfig; + return self(); + } + + public T withConfiguration(Configuration configuration) { + this.serializableConfiguration = new SerializableConfiguration(configuration); + return self(); + } + + @Override + public Buckets createBuckets(int subtaskIndex) throws IOException { + return new Buckets<>( + basePath, + bucketAssigner, + bucketFactory, + new HadoopPathBasedPartFileWriter.HadoopPathBasedBucketWriter<>( + serializableConfiguration.getConfiguration(), + writerFactory, + fileCommitterFactory), + rollingPolicy, + bucketLifeCycleListener, + subtaskIndex, + outputFileConfig); + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/SerializableConfiguration.java b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/SerializableConfiguration.java new file mode 100644 index 0000000000000..536e242694124 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/SerializableConfiguration.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +/** + * Wrapper of hadoop Configuration to make it serializable. + */ +public class SerializableConfiguration implements Serializable { + + private static final long serialVersionUID = 1L; + + private transient Configuration configuration; + + public SerializableConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + public Configuration getConfiguration() { + return configuration; + } + + private void writeObject(ObjectOutputStream out) throws IOException { + configuration.write(out); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + if (configuration == null) { + configuration = new Configuration(); + } + + configuration.readFields(in); + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java new file mode 100644 index 0000000000000..8a4a2a1b57b3b --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.filesystem.HadoopPathBasedBulkFormatBuilder; +import org.apache.flink.streaming.api.functions.sink.filesystem.TestStreamingFileSinkFactory; +import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner; +import org.apache.flink.streaming.util.FiniteTestSource; +import org.apache.flink.test.util.AbstractTestBase; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.IOUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.apache.flink.formats.hadoop.bulk.HadoopPathBasedPartFileWriter.HadoopPathBasedPendingFileRecoverable; +import static org.apache.flink.formats.hadoop.bulk.HadoopPathBasedPartFileWriter.HadoopPathBasedPendingFileRecoverableSerializer; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Base class for testing writing data to the hadoop file system with different configurations. + */ +public class HadoopPathBasedPartFileWriterTest extends AbstractTestBase { + @Rule + public final Timeout timeoutPerTest = Timeout.seconds(2000); + + @Test + public void testPendingFileRecoverableSerializer() throws IOException { + HadoopPathBasedPendingFileRecoverable recoverable = new HadoopPathBasedPendingFileRecoverable( + new Path("hdfs://fake/path")); + HadoopPathBasedPendingFileRecoverableSerializer serializer = + new HadoopPathBasedPendingFileRecoverableSerializer(); + + byte[] serializedBytes = serializer.serialize(recoverable); + HadoopPathBasedPendingFileRecoverable deSerialized = serializer.deserialize( + serializer.getVersion(), + serializedBytes); + + assertEquals(recoverable.getPath(), deSerialized.getPath()); + } + + @Test + public void testWriteFile() throws Exception { + File file = TEMPORARY_FOLDER.newFolder(); + Path basePath = new Path(file.toURI()); + + List data = Arrays.asList( + "first line", + "second line", + "third line"); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + env.enableCheckpointing(100); + + DataStream stream = env.addSource( + new FiniteTestSource<>(data), TypeInformation.of(String.class)); + Configuration configuration = new Configuration(); + + HadoopPathBasedBulkFormatBuilder builder = + new HadoopPathBasedBulkFormatBuilder<>( + basePath, + new TestHadoopPathBasedBulkWriterFactory(), + configuration, + new DateTimeBucketAssigner<>()); + TestStreamingFileSinkFactory streamingFileSinkFactory = new TestStreamingFileSinkFactory<>(); + stream.addSink(streamingFileSinkFactory.createSink(builder, 1000)); + + env.execute(); + validateResult(data, configuration, basePath); + } + + // ------------------------------------------------------------------------ + + private void validateResult(List expected, Configuration config, Path basePath) throws IOException { + FileSystem fileSystem = FileSystem.get(basePath.toUri(), config); + FileStatus[] buckets = fileSystem.listStatus(basePath); + assertNotNull(buckets); + assertEquals(1, buckets.length); + + FileStatus[] partFiles = fileSystem.listStatus(buckets[0].getPath()); + assertNotNull(partFiles); + assertEquals(2, partFiles.length); + + for (FileStatus partFile : partFiles) { + assertTrue(partFile.getLen() > 0); + + List fileContent = readHadoopPath(fileSystem, partFile.getPath()); + assertEquals(expected, fileContent); + } + } + + private List readHadoopPath(FileSystem fileSystem, Path partFile) throws IOException { + try (FSDataInputStream dataInputStream = fileSystem.open(partFile)) { + List lines = new ArrayList<>(); + BufferedReader reader = new BufferedReader(new InputStreamReader(dataInputStream)); + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + + return lines; + } + } + + private static class TestHadoopPathBasedBulkWriterFactory implements HadoopPathBasedBulkWriter.Factory { + + @Override + public HadoopPathBasedBulkWriter create(Path targetFilePath, Path inProgressFilePath) { + try { + FileSystem fileSystem = FileSystem.get(inProgressFilePath.toUri(), new Configuration()); + FSDataOutputStream output = fileSystem.create(inProgressFilePath); + return new FSDataOutputStreamBulkWriterHadoop(output); + } catch (IOException e) { + ExceptionUtils.rethrow(e); + } + + return null; + } + } + + private static class FSDataOutputStreamBulkWriterHadoop implements HadoopPathBasedBulkWriter { + private final FSDataOutputStream outputStream; + + public FSDataOutputStreamBulkWriterHadoop(FSDataOutputStream outputStream) { + this.outputStream = outputStream; + } + + @Override + public long getSize() throws IOException { + return outputStream.getPos(); + } + + @Override + public void dispose() { + IOUtils.closeQuietly(outputStream); + } + + @Override + public void addElement(String element) throws IOException { + outputStream.writeBytes(element + "\n"); + } + + @Override + public void flush() throws IOException { + outputStream.flush(); + } + + @Override + public void finish() throws IOException { + outputStream.flush(); + outputStream.close(); + } + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitterTest.java b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitterTest.java new file mode 100644 index 0000000000000..95c4af317cac7 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/committer/HadoopRenameFileCommitterTest.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.hadoop.bulk.committer; + +import org.apache.flink.formats.hadoop.bulk.HadoopFileCommitter; +import org.apache.flink.test.util.AbstractTestBase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests the behaviors of {@link HadoopRenameFileCommitter}. + */ +public class HadoopRenameFileCommitterTest extends AbstractTestBase { + + private static final List CONTENTS = new ArrayList<>(Arrays.asList( + "first line", + "second line", + "third line")); + + @Test + public void testCommitOneFile() throws IOException { + Configuration configuration = new Configuration(); + + Path basePath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + Path targetFilePath = new Path(basePath, "part-0-0.txt"); + + HadoopFileCommitter committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + writeFile(committer.getInProgressFilePath(), configuration); + + committer.preCommit(); + verifyFileNotExists(configuration, basePath, "part-0-0.txt"); + + committer.commit(); + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt"); + } + + @Test + public void testCommitReWrittenFileAfterFailOver() throws IOException { + Configuration configuration = new Configuration(); + + Path basePath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + Path targetFilePath = new Path(basePath, "part-0-0.txt"); + + HadoopFileCommitter committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + writeFile(committer.getInProgressFilePath(), configuration); + + // Simulates restart the process and re-write the file. + committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + writeFile(committer.getInProgressFilePath(), configuration); + + committer.preCommit(); + verifyFileNotExists(configuration, basePath, "part-0-0.txt"); + + committer.commit(); + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt"); + } + + @Test + public void testCommitPreCommittedFileAfterFailOver() throws IOException { + Configuration configuration = new Configuration(); + + Path basePath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + Path targetFilePath = new Path(basePath, "part-0-0.txt"); + + HadoopFileCommitter committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + writeFile(committer.getInProgressFilePath(), configuration); + + committer.preCommit(); + verifyFileNotExists(configuration, basePath, "part-0-0.txt"); + + // Simulates restart the process and continue committing the file. + committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + committer.commit(); + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt"); + } + + @Test + public void testRepeatCommitAfterFailOver() throws IOException { + Configuration configuration = new Configuration(); + + Path basePath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + Path targetFilePath = new Path(basePath, "part-0-0.txt"); + + HadoopFileCommitter committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + writeFile(committer.getInProgressFilePath(), configuration); + + committer.preCommit(); + verifyFileNotExists(configuration, basePath, "part-0-0.txt"); + + committer.commit(); + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt"); + + // Simulates restart the process and continue committing the file. + committer = new HadoopRenameFileCommitter(configuration, targetFilePath); + committer.commitAfterRecovery(); + + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt"); + } + + @Test + public void testCommitMultipleFilesOneByOne() throws IOException { + Configuration configuration = new Configuration(); + + Path basePath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + Path targetFilePath1 = new Path(basePath, "part-0-0.txt"); + Path targetFilePath2 = new Path(basePath, "part-1-1.txt"); + + HadoopFileCommitter committer1 = new HadoopRenameFileCommitter(configuration, targetFilePath1); + HadoopFileCommitter committer2 = new HadoopRenameFileCommitter(configuration, targetFilePath2); + + writeFile(committer1.getInProgressFilePath(), configuration); + writeFile(committer2.getInProgressFilePath(), configuration); + + committer1.preCommit(); + committer1.commit(); + + verifyCommittedFiles(configuration, basePath, "part-0-0.txt"); + verifyFileNotExists(configuration, basePath, "part-1-1.txt"); + + committer2.preCommit(); + committer2.commit(); + + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt", "part-1-1.txt"); + } + + @Test + public void testCommitMultipleFilesMixed() throws IOException { + Configuration configuration = new Configuration(); + + Path basePath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + Path targetFilePath1 = new Path(basePath, "part-0-0.txt"); + Path targetFilePath2 = new Path(basePath, "part-1-1.txt"); + + HadoopFileCommitter committer1 = new HadoopRenameFileCommitter(configuration, targetFilePath1); + HadoopFileCommitter committer2 = new HadoopRenameFileCommitter(configuration, targetFilePath2); + + writeFile(committer1.getInProgressFilePath(), configuration); + writeFile(committer2.getInProgressFilePath(), configuration); + + committer1.preCommit(); + committer2.preCommit(); + + verifyFileNotExists(configuration, basePath, "part-0-0.txt"); + verifyFileNotExists(configuration, basePath, "part-1-1.txt"); + + committer1.commit(); + verifyCommittedFiles(configuration, basePath, "part-0-0.txt"); + verifyFileNotExists(configuration, basePath, "part-1-1.txt"); + + committer2.commit(); + verifyFolderAfterAllCommitted(configuration, basePath, "part-0-0.txt", "part-1-1.txt"); + } + + //--------------------------------------------------------------------------------------- + + private void writeFile(Path path, Configuration configuration) throws IOException { + FileSystem fileSystem = FileSystem.get(path.toUri(), configuration); + try (FSDataOutputStream fsDataOutputStream = fileSystem.create(path, true); + PrintWriter printWriter = new PrintWriter(fsDataOutputStream)) { + + for (String line : CONTENTS) { + printWriter.println(line); + } + } + } + + private void verifyFileNotExists( + Configuration configuration, + Path basePath, + String... targetFileNames) throws IOException { + + FileSystem fileSystem = FileSystem.get(basePath.toUri(), configuration); + for (String targetFileName : targetFileNames) { + assertFalse( + "Pre-committed file should not exists: " + targetFileName, + fileSystem.exists(new Path(basePath, targetFileName))); + } + } + + private void verifyCommittedFiles( + Configuration configuration, + Path basePath, + String... targetFileNames) throws IOException { + + FileSystem fileSystem = FileSystem.get(basePath.toUri(), configuration); + for (String targetFileName : targetFileNames) { + Path targetFilePath = new Path(basePath, targetFileName); + assertTrue( + "Committed file should exists: " + targetFileName, + fileSystem.exists(targetFilePath)); + List written = readFile(fileSystem, targetFilePath); + assertEquals( + "Unexpected file content for file " + targetFilePath, + CONTENTS, + written); + } + } + + private void verifyFolderAfterAllCommitted( + Configuration configuration, + Path basePath, + String... targetFileNames) throws IOException { + + List expectedNames = Arrays.asList(targetFileNames); + Collections.sort(expectedNames); + + FileSystem fileSystem = FileSystem.get(basePath.toUri(), configuration); + FileStatus[] files = fileSystem.listStatus(basePath); + List fileNames = new ArrayList<>(); + for (FileStatus file : files) { + fileNames.add(file.getPath().getName()); + } + Collections.sort(fileNames); + assertEquals( + "Remain files are " + fileNames, + expectedNames, + fileNames); + + for (FileStatus file : files) { + List written = readFile(fileSystem, files[0].getPath()); + assertEquals( + "Unexpected file content for file " + file.getPath(), + CONTENTS, + written); + } + } + + private List readFile(FileSystem fileSystem, Path partFile) throws IOException { + try (FSDataInputStream dataInputStream = fileSystem.open(partFile)) { + List lines = new ArrayList<>(); + BufferedReader reader = new BufferedReader(new InputStreamReader(dataInputStream)); + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + + return lines; + } + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestStreamingFileSinkFactory.java b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestStreamingFileSinkFactory.java new file mode 100644 index 0000000000000..4baea0696ba65 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestStreamingFileSinkFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.filesystem; + +/** + * Factory to create the streaming file sink. + */ +public class TestStreamingFileSinkFactory { + + public StreamingFileSink createSink( + StreamingFileSink.BucketsBuilder> bucketsBuilder, + long bucketCheckInterval) { + + return new StreamingFileSink<>(bucketsBuilder, bucketCheckInterval); + } +} diff --git a/flink-formats/flink-hadoop-bulk/src/test/resources/log4j2-test.properties b/flink-formats/flink-hadoop-bulk/src/test/resources/log4j2-test.properties new file mode 100644 index 0000000000000..835c2ec9a3d02 --- /dev/null +++ b/flink-formats/flink-hadoop-bulk/src/test/resources/log4j2-test.properties @@ -0,0 +1,28 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level = OFF +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.target = SYSTEM_ERR +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n diff --git a/flink-formats/pom.xml b/flink-formats/pom.xml index 68ac3c9ad294b..3952f0c88ca50 100644 --- a/flink-formats/pom.xml +++ b/flink-formats/pom.xml @@ -45,6 +45,7 @@ under the License. flink-csv flink-orc flink-orc-nohive + flink-hadoop-bulk @@ -890,6 +896,7 @@ under the License. org.apache.flink:flink-orc-nohive_${scala.binary.version} org.apache.flink:flink-hadoop-compatibility_${scala.binary.version} org.apache.flink:flink-parquet_${scala.binary.version} + org.apache.flink:flink-hadoop-bulk_${scala.binary.version} org.apache.parquet:parquet-hadoop org.apache.parquet:parquet-format org.apache.parquet:parquet-column diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveOptions.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveOptions.java index bb9729ec75270..161bdaa494a43 100644 --- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveOptions.java +++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveOptions.java @@ -45,4 +45,11 @@ public class HiveOptions { key("table.exec.hive.infer-source-parallelism.max") .defaultValue(1000) .withDescription("Sets max infer parallelism for source operator."); + + public static final ConfigOption TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER = + key("table.exec.hive.fallback-mapred-writer") + .booleanType() + .defaultValue(true) + .withDescription("If it is false, using flink native writer to write parquet and orc files; " + + "If it is true, using hadoop mapred record writer to write parquet and orc files."); } diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableFactory.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableFactory.java index a60460b38888c..01b16c55c3b8b 100644 --- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableFactory.java +++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableFactory.java @@ -87,6 +87,8 @@ public TableSink createTableSink(TableSinkFactory.Context context) { if (!isGeneric) { return new HiveTableSink( + context.getConfiguration().get( + HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER), context.isBounded(), new JobConf(hiveConf), context.getObjectIdentifier(), diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSink.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSink.java index 94c05e4252e14..51f2ec9f3d337 100644 --- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSink.java +++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSink.java @@ -20,15 +20,17 @@ import org.apache.flink.api.common.serialization.BulkWriter; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.connectors.hive.write.HiveBulkWriterFactory; import org.apache.flink.connectors.hive.write.HiveOutputFormatFactory; import org.apache.flink.connectors.hive.write.HiveWriterFactory; import org.apache.flink.formats.parquet.row.ParquetRowDataBuilder; import org.apache.flink.orc.OrcSplitReaderUtil; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.functions.sink.filesystem.HadoopPathBasedBulkFormatBuilder; import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig; import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; -import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink.BulkFormatBuilder; +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink.BucketsBuilder; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.ObjectIdentifier; @@ -68,11 +70,14 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.orc.TypeDescription; import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.apache.flink.table.filesystem.FileSystemTableFactory.SINK_ROLLING_POLICY_FILE_SIZE; import static org.apache.flink.table.filesystem.FileSystemTableFactory.SINK_ROLLING_POLICY_TIME_INTERVAL; @@ -82,6 +87,9 @@ */ public class HiveTableSink implements AppendStreamTableSink, PartitionableTableSink, OverwritableTableSink { + private static final Logger LOG = LoggerFactory.getLogger(HiveTableSink.class); + + private final boolean userMrWriter; private final boolean isBounded; private final JobConf jobConf; private final CatalogTable catalogTable; @@ -95,7 +103,9 @@ public class HiveTableSink implements AppendStreamTableSink, PartitionableTableS private boolean overwrite = false; private boolean dynamicGrouping = false; - public HiveTableSink(boolean isBounded, JobConf jobConf, ObjectIdentifier identifier, CatalogTable table) { + public HiveTableSink( + boolean userMrWriter, boolean isBounded, JobConf jobConf, ObjectIdentifier identifier, CatalogTable table) { + this.userMrWriter = userMrWriter; this.isBounded = isBounded; this.jobConf = jobConf; this.identifier = identifier; @@ -130,6 +140,11 @@ public final DataStreamSink consumeDataStream(DataStream dataStream) { HiveReflectionUtils.getTableMetadata(hiveShim, table), hiveShim, isCompressed); + String extension = Utilities.getFileExtension(jobConf, isCompressed, + (HiveOutputFormat) hiveOutputFormatClz.newInstance()); + extension = extension == null ? "" : extension; + OutputFileConfig outputFileConfig = OutputFileConfig.builder() + .withPartSuffix(extension).build(); if (isBounded) { FileSystemOutputFormat.Builder builder = new FileSystemOutputFormat.Builder<>(); builder.setPartitionComputer(new HiveRowPartitionComputer( @@ -150,17 +165,11 @@ public final DataStreamSink consumeDataStream(DataStream dataStream) { builder.setStaticPartitions(staticPartitionSpec); builder.setTempPath(new org.apache.flink.core.fs.Path( toStagingDir(sd.getLocation(), jobConf))); - String extension = Utilities.getFileExtension(jobConf, isCompressed, - (HiveOutputFormat) hiveOutputFormatClz.newInstance()); - extension = extension == null ? "" : extension; - OutputFileConfig outputFileConfig = new OutputFileConfig("", extension); builder.setOutputFileConfig(outputFileConfig); - return dataStream .writeUsingOutputFormat(builder.build()) .setParallelism(dataStream.getParallelism()); } else { - BulkWriter.Factory bulkFactory = createBulkWriterFactory(partitionColumns, sd); org.apache.flink.configuration.Configuration conf = new org.apache.flink.configuration.Configuration(); catalogTable.getOptions().forEach(conf::setString); HiveRowDataPartitionComputer partComputer = new HiveRowDataPartitionComputer( @@ -178,12 +187,26 @@ public final DataStreamSink consumeDataStream(DataStream dataStream) { conf.get(SINK_ROLLING_POLICY_TIME_INTERVAL)); InactiveBucketListener listener = new InactiveBucketListener(); - BulkFormatBuilder builder = StreamingFileSink.forBulkFormat( - new org.apache.flink.core.fs.Path(sd.getLocation()), - new FileSystemTableSink.ProjectionBulkFactory(bulkFactory, partComputer)) - .withBucketAssigner(assigner) - .withBucketLifeCycleListener(listener) - .withRollingPolicy(rollingPolicy); + Optional> bulkFactory = createBulkWriterFactory(partitionColumns, sd); + BucketsBuilder> builder; + if (userMrWriter || !bulkFactory.isPresent()) { + HiveBulkWriterFactory hadoopBulkFactory = new HiveBulkWriterFactory(recordWriterFactory); + builder = new HadoopPathBasedBulkFormatBuilder<>( + new Path(sd.getLocation()), hadoopBulkFactory, jobConf, assigner) + .withRollingPolicy(rollingPolicy) + .withBucketLifeCycleListener(listener) + .withOutputFileConfig(outputFileConfig); + LOG.info("Hive streaming sink: Use MapReduce RecordWriter writer."); + } else { + builder = StreamingFileSink.forBulkFormat( + new org.apache.flink.core.fs.Path(sd.getLocation()), + new FileSystemTableSink.ProjectionBulkFactory(bulkFactory.get(), partComputer)) + .withBucketAssigner(assigner) + .withBucketLifeCycleListener(listener) + .withRollingPolicy(rollingPolicy) + .withOutputFileConfig(outputFileConfig); + LOG.info("Hive streaming sink: Use native parquet&orc writer."); + } return FileSystemTableSink.createStreamingSink( conf, new org.apache.flink.core.fs.Path(sd.getLocation()), @@ -206,7 +229,7 @@ public final DataStreamSink consumeDataStream(DataStream dataStream) { } } - private BulkWriter.Factory createBulkWriterFactory(String[] partitionColumns, + private Optional> createBulkWriterFactory(String[] partitionColumns, StorageDescriptor sd) { String serLib = sd.getSerdeInfo().getSerializationLib().toLowerCase(); int formatFieldCount = tableSchema.getFieldCount() - partitionColumns.length; @@ -219,20 +242,16 @@ private BulkWriter.Factory createBulkWriterFactory(String[] partitionCo RowType formatType = RowType.of(formatTypes, formatNames); Configuration formatConf = new Configuration(jobConf); sd.getSerdeInfo().getParameters().forEach(formatConf::set); - BulkWriter.Factory bulkFactory; if (serLib.contains("parquet")) { - bulkFactory = ParquetRowDataBuilder.createWriterFactory( - formatType, formatConf, hiveVersion.startsWith("3.")); + return Optional.of(ParquetRowDataBuilder.createWriterFactory( + formatType, formatConf, hiveVersion.startsWith("3."))); } else if (serLib.contains("orc")) { TypeDescription typeDescription = OrcSplitReaderUtil.logicalTypeToOrcType(formatType); - bulkFactory = hiveShim.createOrcBulkWriterFactory( - formatConf, typeDescription.toString(), formatTypes); + return Optional.of(hiveShim.createOrcBulkWriterFactory( + formatConf, typeDescription.toString(), formatTypes)); } else { - throw new UnsupportedOperationException(String.format( - "Only parquet or orc can support streaming writing, but now serialization lib is %s.", - serLib)); + return Optional.empty(); } - return bulkFactory; } @Override diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/write/HiveBulkWriterFactory.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/write/HiveBulkWriterFactory.java new file mode 100644 index 0000000000000..9a22d5662e60b --- /dev/null +++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/write/HiveBulkWriterFactory.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connectors.hive.write; + +import org.apache.flink.formats.hadoop.bulk.HadoopPathBasedBulkWriter; +import org.apache.flink.table.data.RowData; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; +import org.apache.hadoop.io.Writable; + +import java.io.IOException; +import java.util.function.Function; + +/** + * Hive bulk writer factory for path-based bulk file writer that writes to the specific hadoop path. + */ +public class HiveBulkWriterFactory implements HadoopPathBasedBulkWriter.Factory { + + private static final long serialVersionUID = 1L; + + private final HiveWriterFactory factory; + + public HiveBulkWriterFactory(HiveWriterFactory factory) { + this.factory = factory; + } + + @Override + public HadoopPathBasedBulkWriter create(Path targetPath, Path inProgressPath) throws IOException { + FileSinkOperator.RecordWriter recordWriter = factory.createRecordWriter(inProgressPath); + Function rowConverter = factory.createRowDataConverter(); + FileSystem fs = FileSystem.get(inProgressPath.toUri(), factory.getJobConf()); + return new HadoopPathBasedBulkWriter() { + + @Override + public long getSize() throws IOException { + return fs.getFileStatus(inProgressPath).getLen(); + } + + @Override + public void dispose() { + // close silently. + try { + recordWriter.close(true); + } catch (IOException ignored) { + } + } + + @Override + public void addElement(RowData element) throws IOException { + recordWriter.write(rowConverter.apply(element)); + } + + @Override + public void flush() { + } + + @Override + public void finish() throws IOException { + recordWriter.close(false); + } + }; + } +} diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java index 4006d8215a702..b823a9bef0fdf 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java @@ -228,25 +228,46 @@ public void testWriteNullValues() throws Exception { } } + @Test(timeout = 120000) + public void testDefaultSerPartStreamingWrite() throws Exception { + testStreamingWrite(true, false, true, this::checkSuccessFiles); + } + @Test(timeout = 120000) public void testPartStreamingWrite() throws Exception { - testStreamingWrite(true, (path) -> { - File basePath = new File(path, "d=2020-05-03"); - Assert.assertEquals(5, basePath.list().length); - Assert.assertTrue(new File(new File(basePath, "e=7"), "_MY_SUCCESS").exists()); - Assert.assertTrue(new File(new File(basePath, "e=8"), "_MY_SUCCESS").exists()); - Assert.assertTrue(new File(new File(basePath, "e=9"), "_MY_SUCCESS").exists()); - Assert.assertTrue(new File(new File(basePath, "e=10"), "_MY_SUCCESS").exists()); - Assert.assertTrue(new File(new File(basePath, "e=11"), "_MY_SUCCESS").exists()); - }); + testStreamingWrite(true, false, false, this::checkSuccessFiles); } @Test(timeout = 120000) public void testNonPartStreamingWrite() throws Exception { - testStreamingWrite(false, (p) -> {}); + testStreamingWrite(false, false, false, (p) -> {}); + } + + @Test(timeout = 120000) + public void testPartStreamingMrWrite() throws Exception { + testStreamingWrite(true, true, false, this::checkSuccessFiles); } - private void testStreamingWrite(boolean part, Consumer pathConsumer) throws Exception { + @Test(timeout = 120000) + public void testNonPartStreamingMrWrite() throws Exception { + testStreamingWrite(false, true, false, (p) -> {}); + } + + private void checkSuccessFiles(String path) { + File basePath = new File(path, "d=2020-05-03"); + Assert.assertEquals(5, basePath.list().length); + Assert.assertTrue(new File(new File(basePath, "e=7"), "_MY_SUCCESS").exists()); + Assert.assertTrue(new File(new File(basePath, "e=8"), "_MY_SUCCESS").exists()); + Assert.assertTrue(new File(new File(basePath, "e=9"), "_MY_SUCCESS").exists()); + Assert.assertTrue(new File(new File(basePath, "e=10"), "_MY_SUCCESS").exists()); + Assert.assertTrue(new File(new File(basePath, "e=11"), "_MY_SUCCESS").exists()); + } + + private void testStreamingWrite( + boolean part, + boolean useMr, + boolean defaultSer, + Consumer pathConsumer) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); env.enableCheckpointing(100); @@ -255,6 +276,13 @@ private void testStreamingWrite(boolean part, Consumer pathConsumer) thr tEnv.registerCatalog(hiveCatalog.getName(), hiveCatalog); tEnv.useCatalog(hiveCatalog.getName()); tEnv.getConfig().setSqlDialect(SqlDialect.HIVE); + if (useMr) { + tEnv.getConfig().getConfiguration().set( + HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER, true); + } else { + tEnv.getConfig().getConfiguration().set( + HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER, false); + } try { tEnv.executeSql("create database db1"); @@ -277,7 +305,8 @@ private void testStreamingWrite(boolean part, Consumer pathConsumer) thr (part ? "" : ",d string,e string") + ") " + (part ? "partitioned by (d string,e string) " : "") + - " stored as parquet TBLPROPERTIES (" + + (defaultSer ? "" : " stored as parquet") + + " TBLPROPERTIES (" + "'" + PARTITION_TIME_EXTRACTOR_TIMESTAMP_PATTERN.key() + "'='$d $e:00:00'," + "'" + SINK_PARTITION_COMMIT_DELAY.key() + "'='1h'," + "'" + SINK_PARTITION_COMMIT_POLICY_KIND.key() + "'='metastore,success-file'," + diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/TableEnvHiveConnectorTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/TableEnvHiveConnectorTest.java index 9aca18cdee1c5..7f6a78d1a9b3d 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/TableEnvHiveConnectorTest.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/TableEnvHiveConnectorTest.java @@ -18,6 +18,7 @@ package org.apache.flink.connectors.hive; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.HiveVersionTestUtil; import org.apache.flink.table.api.StatementSet; import org.apache.flink.table.api.TableEnvironment; @@ -536,8 +537,7 @@ public void testWhitespacePartValue() throws Exception { } } - @Test - public void testCompressTextTable() throws Exception { + private void testCompressTextTable(boolean batch) throws Exception { hiveShell.execute("create database db1"); try { hiveShell.execute("create table db1.src (x string,y string)"); @@ -547,8 +547,10 @@ public void testCompressTextTable() throws Exception { .addRow(new Object[]{"c", "d"}) .commit(); hiveCatalog.getHiveConf().setBoolVar(HiveConf.ConfVars.COMPRESSRESULT, true); - TableEnvironment tableEnv = getTableEnvWithHiveCatalog(); - TableEnvUtil.execInsertSqlAndWaitResult(tableEnv, "insert overwrite db1.dest select * from db1.src"); + TableEnvironment tableEnv = batch ? + getTableEnvWithHiveCatalog() : + getStreamTableEnvWithHiveCatalog(); + TableEnvUtil.execInsertSqlAndWaitResult(tableEnv, "insert into db1.dest select * from db1.src"); List expected = Arrays.asList("a\tb", "c\td"); verifyHiveQueryResult("select * from db1.dest", expected); verifyFlinkQueryResult(tableEnv.sqlQuery("select * from db1.dest"), expected); @@ -557,6 +559,16 @@ public void testCompressTextTable() throws Exception { } } + @Test + public void testBatchCompressTextTable() throws Exception { + testCompressTextTable(true); + } + + @Test + public void testStreamCompressTextTable() throws Exception { + testCompressTextTable(false); + } + @Test public void testRegexSerDe() throws Exception { hiveShell.execute("create database db1"); @@ -638,6 +650,14 @@ private TableEnvironment getTableEnvWithHiveCatalog() { return tableEnv; } + private TableEnvironment getStreamTableEnvWithHiveCatalog() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + TableEnvironment tableEnv = HiveTestUtils.createTableEnvWithBlinkPlannerStreamMode(env); + tableEnv.registerCatalog(hiveCatalog.getName(), hiveCatalog); + tableEnv.useCatalog(hiveCatalog.getName()); + return tableEnv; + } + private void verifyHiveQueryResult(String query, List expected) { List results = hiveShell.executeQuery(query); assertEquals(expected.size(), results.size()); From 4314e7af4bfc46f8f61dd58a1b2a9350e91c38fb Mon Sep 17 00:00:00 2001 From: libenchao Date: Sun, 23 Feb 2020 15:21:29 +0800 Subject: [PATCH 027/773] [FLINK-16094][docs-zh] Translate /dev/table/functions/udfs.zh.md into Chinese This closes #11191 --- docs/dev/table/functions/udfs.md | 101 ++++---- docs/dev/table/functions/udfs.zh.md | 388 ++++++++++++++-------------- 2 files changed, 237 insertions(+), 252 deletions(-) diff --git a/docs/dev/table/functions/udfs.md b/docs/dev/table/functions/udfs.md index 26f557a54f570..c53a25826dd78 100644 --- a/docs/dev/table/functions/udfs.md +++ b/docs/dev/table/functions/udfs.md @@ -215,6 +215,30 @@ tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(s {% endhighlight %} +Please note that POJO types do not have a deterministic field order. Therefore, you cannot rename the fields of POJO returned by a table function using `AS`. + +By default the result type of a `TableFunction` is determined by Flink’s automatic type extraction facilities. This works well for basic types and simple POJOs but might be wrong for more complex, custom, or composite types. In such a case, the type of the result can be manually specified by overriding `TableFunction#getResultType()` which returns its `TypeInformation`. + +The following example shows an example of a `TableFunction` that returns a `Row` type which requires explicit type information. We define that the returned table type should be `RowTypeInfo(String, Integer)` by overriding `TableFunction#getResultType()`. + +{% highlight java %} +public class CustomTypeSplit extends TableFunction { + public void eval(String str) { + for (String s : str.split(" ")) { + Row row = new Row(2); + row.setField(0, s); + row.setField(1, s.length()); + collect(row); + } + } + + @Override + public TypeInformation getResultType() { + return Types.ROW(Types.STRING(), Types.INT()); + } +} +{% endhighlight %} +
In order to define a table function one has to extend the base class `TableFunction` in `org.apache.flink.table.functions` and implement (one or more) evaluation methods. The behavior of a table function is determined by its evaluation methods. An evaluation method must be declared `public` and named `eval`. The `TableFunction` can be overloaded by implementing multiple methods named `eval`. The parameter types of the evaluation methods determine all valid parameters of the table function. Evaluation methods can also support variable arguments, such as `eval(String... strs)`. The type of the returned table is determined by the generic type of `TableFunction`. Evaluation methods emit output rows using the protected `collect(T)` method. @@ -250,6 +274,30 @@ tableEnv.sqlQuery("SELECT a, word, length FROM MyTable, LATERAL TABLE(split(a)) tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(split(a)) as T(word, length) ON TRUE") {% endhighlight %} **IMPORTANT:** Do not implement TableFunction as a Scala object. Scala object is a singleton and will cause concurrency issues. + +Please note that POJO types do not have a deterministic field order. Therefore, you cannot rename the fields of POJO returned by a table function using `AS`. + +By default the result type of a `TableFunction` is determined by Flink’s automatic type extraction facilities. This works well for basic types and simple POJOs but might be wrong for more complex, custom, or composite types. In such a case, the type of the result can be manually specified by overriding `TableFunction#getResultType()` which returns its `TypeInformation`. + +The following example shows an example of a `TableFunction` that returns a `Row` type which requires explicit type information. We define that the returned table type should be `RowTypeInfo(String, Integer)` by overriding `TableFunction#getResultType()`. + +{% highlight scala %} +class CustomTypeSplit extends TableFunction[Row] { + def eval(str: String): Unit = { + str.split(" ").foreach({ s => + val row = new Row(2) + row.setField(0, s) + row.setField(1, s.length) + collect(row) + }) + } + + override def getResultType: TypeInformation[Row] = { + Types.ROW(Types.STRING, Types.INT) + } +} +{% endhighlight %} +
@@ -288,59 +336,6 @@ Please refer to the [Python Table Function]({{ site.baseurl }}/dev/table/python/
-
-
-Please note that POJO types do not have a deterministic field order. Therefore, you cannot rename the fields of POJO returned by a table function using `AS`. - -By default the result type of a `TableFunction` is determined by Flink’s automatic type extraction facilities. This works well for basic types and simple POJOs but might be wrong for more complex, custom, or composite types. In such a case, the type of the result can be manually specified by overriding `TableFunction#getResultType()` which returns its `TypeInformation`. - -The following example shows an example of a `TableFunction` that returns a `Row` type which requires explicit type information. We define that the returned table type should be `RowTypeInfo(String, Integer)` by overriding `TableFunction#getResultType()`. - -{% highlight java %} -public class CustomTypeSplit extends TableFunction { - public void eval(String str) { - for (String s : str.split(" ")) { - Row row = new Row(2); - row.setField(0, s); - row.setField(1, s.length()); - collect(row); - } - } - - @Override - public TypeInformation getResultType() { - return Types.ROW(Types.STRING(), Types.INT()); - } -} -{% endhighlight %} -
- -
-Please note that POJO types do not have a deterministic field order. Therefore, you cannot rename the fields of POJO returned by a table function using `AS`. - -By default the result type of a `TableFunction` is determined by Flink’s automatic type extraction facilities. This works well for basic types and simple POJOs but might be wrong for more complex, custom, or composite types. In such a case, the type of the result can be manually specified by overriding `TableFunction#getResultType()` which returns its `TypeInformation`. - -The following example shows an example of a `TableFunction` that returns a `Row` type which requires explicit type information. We define that the returned table type should be `RowTypeInfo(String, Integer)` by overriding `TableFunction#getResultType()`. - -{% highlight scala %} -class CustomTypeSplit extends TableFunction[Row] { - def eval(str: String): Unit = { - str.split(" ").foreach({ s => - val row = new Row(2) - row.setField(0, s) - row.setField(1, s.length) - collect(row) - }) - } - - override def getResultType: TypeInformation[Row] = { - Types.ROW(Types.STRING, Types.INT) - } -} -{% endhighlight %} -
-
- {% top %} diff --git a/docs/dev/table/functions/udfs.zh.md b/docs/dev/table/functions/udfs.zh.md index c0e7c67a20bbd..995a67cd727c9 100644 --- a/docs/dev/table/functions/udfs.zh.md +++ b/docs/dev/table/functions/udfs.zh.md @@ -22,33 +22,31 @@ specific language governing permissions and limitations under the License. --> -User-defined functions are an important feature, because they significantly extend the expressiveness of queries. +自定义函数是一个非常重要的功能，因为它极大的扩展了查询的表达能力。 * This will be replaced by the TOC {:toc} -Register User-Defined Functions +注册自定义函数 ------------------------------- -In most cases, a user-defined function must be registered before it can be used in an query. It is not necessary to register functions for the Scala Table API. +在大多数情况下，自定义函数在使用之前都需要注册。在 Scala Table API 中可以不用注册。 -Functions are registered at the `TableEnvironment` by calling a `registerFunction()` method. When a user-defined function is registered, it is inserted into the function catalog of the `TableEnvironment` such that the Table API or SQL parser can recognize and properly translate it. - -Please find detailed examples of how to register and how to call each type of user-defined function -(`ScalarFunction`, `TableFunction`, and `AggregateFunction`) in the following sub-sessions. +通过调用 `registerFunction()` 把函数注册到 `TableEnvironment`。当一个函数注册之后，它就在 `TableEnvironment` 的函数 catalog 里面了，这样 Table API 或者 SQL 解析器就可以识别并使用它。 +关于如何注册和使用每种类型的自定义函数（标量函数、表值函数和聚合函数），更多示例可以看下面的部分。 {% top %} -Scalar Functions +标量函数 ---------------- -If a required scalar function is not contained in the built-in functions, it is possible to define custom, user-defined scalar functions for both the Table API and SQL. A user-defined scalar functions maps zero, one, or multiple scalar values to a new scalar value. +如果需要的标量函数没有被内置函数覆盖，就可以在自定义一个标量函数在 Table API 和 SQL 中使用。自定义标量函数可以把 0 到多个标量值映射成 1 个标量值。
-In order to define a scalar function, one has to extend the base class `ScalarFunction` in `org.apache.flink.table.functions` and implement (one or more) evaluation methods. The behavior of a scalar function is determined by the evaluation method. An evaluation method must be declared publicly and named `eval`. The parameter types and return type of the evaluation method also determine the parameter and return types of the scalar function. Evaluation methods can also be overloaded by implementing multiple methods named `eval`. Evaluation methods can also support variable arguments, such as `eval(String... strs)`. +想要实现自定义标量函数，你需要扩展 `org.apache.flink.table.functions` 里面的 `ScalarFunction` 并且实现一个或者多个求值方法。标量函数的行为取决于你写的求值方法。求值方法并须是 `public` 的，而且名字必须是 `eval`。求值方法的参数类型以及返回值类型就决定了标量函数的参数类型和返回值类型。可以通过实现多个名为 `eval` 的方法对求值方法进行重载。求值方法也支持可变参数，例如 `eval(String... strs)`。 -The following example shows how to define your own hash code function, register it in the TableEnvironment, and call it in a query. Note that you can configure your scalar function via a constructor before it is registered: +下面的示例展示了如何实现一个求哈希值的函数。先把它注册到 `TableEnvironment` 里，然后在查询的时候就可以直接使用了。需要注意的是，你可以在注册之前通过构造方法来配置你的标量函数： {% highlight java %} public class HashCode extends ScalarFunction { @@ -65,19 +63,19 @@ public class HashCode extends ScalarFunction { BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env); -// register the function +// 注册函数 tableEnv.registerFunction("hashCode", new HashCode(10)); -// use the function in Java Table API +// 在 Java Table API 中使用函数 myTable.select("string, string.hashCode(), hashCode(string)"); -// use the function in SQL API +// 在 SQL API 中使用函数 tableEnv.sqlQuery("SELECT string, hashCode(string) FROM MyTable"); {% endhighlight %} -By default the result type of an evaluation method is determined by Flink's type extraction facilities. This is sufficient for basic types or simple POJOs but might be wrong for more complex, custom, or composite types. In these cases `TypeInformation` of the result type can be manually defined by overriding `ScalarFunction#getResultType()`. +求值方法的返回值类型默认是由 Flink 的类型推导来决定的。类型推导可以推导出基本数据类型以及简单的 POJO，但是对于更复杂的、自定义的、或者组合类型，可能会推导出错误的结果。在这种情况下，可以通过覆盖 `ScalarFunction#getResultType()`，并且返回 `TypeInformation` 来定义复杂类型。 -The following example shows an advanced example which takes the internal timestamp representation and also returns the internal timestamp representation as a long value. By overriding `ScalarFunction#getResultType()` we define that the returned long value should be interpreted as a `Types.TIMESTAMP` by the code generation. +下面的示例展示了一个高级一点的自定义标量函数用法，它接收一个内部的时间戳参数，并且以 `long` 的形式返回该内部的时间戳。通过覆盖 `ScalarFunction#getResultType()`，我们定义了我们返回的 `long` 类型在代码生成时可以被解析为 `Types.TIMESTAMP` 类型。 {% highlight java %} public static class TimestampModifier extends ScalarFunction { @@ -93,12 +91,12 @@ public static class TimestampModifier extends ScalarFunction {
-In order to define a scalar function, one has to extend the base class `ScalarFunction` in `org.apache.flink.table.functions` and implement (one or more) evaluation methods. The behavior of a scalar function is determined by the evaluation method. An evaluation method must be declared publicly and named `eval`. The parameter types and return type of the evaluation method also determine the parameter and return types of the scalar function. Evaluation methods can also be overloaded by implementing multiple methods named `eval`. Evaluation methods can also support variable arguments, such as `@varargs def eval(str: String*)`. +想要实现自定义标量函数，你需要扩展 `org.apache.flink.table.functions` 里面的 `ScalarFunction` 并且实现一个或者多个求值方法。标量函数的行为取决于你写的求值方法。求值方法并须是 `public` 的，而且名字必须是 `eval`。求值方法的参数类型以及返回值类型就决定了标量函数的参数类型和返回值类型。可以通过实现多个名为 `eval` 的方法对求值方法进行重载。求值方法也支持可变参数，例如 `@varargs def eval(str: String*)`。 -The following example shows how to define your own hash code function, register it in the TableEnvironment, and call it in a query. Note that you can configure your scalar function via a constructor before it is registered: +下面的示例展示了如何实现一个求哈希值的函数。先把它注册到 `TableEnvironment` 里，然后在查询的时候就可以直接使用了。需要注意的是，你可以在注册之前通过构造方法来配置你的标量函数： {% highlight scala %} -// must be defined in static/object context +// 必须定义在 static/object 上下文中 class HashCode(factor: Int) extends ScalarFunction { def eval(s: String): Int = { s.hashCode() * factor @@ -107,18 +105,18 @@ class HashCode(factor: Int) extends ScalarFunction { val tableEnv = BatchTableEnvironment.create(env) -// use the function in Scala Table API +// 在 Scala Table API 中使用函数 val hashCode = new HashCode(10) myTable.select('string, hashCode('string)) -// register and use the function in SQL +// 在 SQL 中注册和使用函数 tableEnv.registerFunction("hashCode", new HashCode(10)) tableEnv.sqlQuery("SELECT string, hashCode(string) FROM MyTable") {% endhighlight %} -By default the result type of an evaluation method is determined by Flink's type extraction facilities. This is sufficient for basic types or simple POJOs but might be wrong for more complex, custom, or composite types. In these cases `TypeInformation` of the result type can be manually defined by overriding `ScalarFunction#getResultType()`. +求值方法的返回值类型默认是由 Flink 的类型推导来决定的。类型推导可以推导出基本数据类型以及简单的 POJO，但是对于更复杂的、自定义的、或者组合类型，可能会推导出错误的结果。在这种情况下，可以通过覆盖 `ScalarFunction#getResultType()`，并且返回 `TypeInformation` 来定义复杂类型。 -The following example shows an advanced example which takes the internal timestamp representation and also returns the internal timestamp representation as a long value. By overriding `ScalarFunction#getResultType()` we define that the returned long value should be interpreted as a `Types.TIMESTAMP` by the code generation. +下面的示例展示了一个高级一点的自定义标量函数用法，它接收一个内部的时间戳参数，并且以 `long` 的形式返回该内部的时间戳。通过覆盖 `ScalarFunction#getResultType()`，我们定义了我们返回的 `long` 类型在代码生成时可以被解析为 `Types.TIMESTAMP` 类型。 {% highlight scala %} object TimestampModifier extends ScalarFunction { @@ -134,9 +132,9 @@ object TimestampModifier extends ScalarFunction {
-In order to define a Python scalar function, one can extend the base class `ScalarFunction` in `pyflink.table.udf` and implement an evaluation method. The behavior of a Python scalar function is determined by the evaluation method which is named `eval`. +要定义一个 Python 标量函数，你可以继承 `pyflink.table.udf` 下的 `ScalarFunction`，并且实现一个求值函数。Python 标量函数的行为取决于你实现的求值函数，它的名字必须是 `eval`。 -The following example shows how to define your own Python hash code function, register it in the TableEnvironment, and call it in a query. Note that you can configure your scalar function via a constructor before it is registered: +下面的示例展示了如何自定义一个 Python 的求哈希值的函数，并且把它注册到 `TableEnvironment` 里，然后在查询中使用它。你可以在注册函数之前通过构造函数来配置你的标量函数。 {% highlight python %} class HashCode(ScalarFunction): @@ -148,39 +146,39 @@ class HashCode(ScalarFunction): table_env = BatchTableEnvironment.create(env) -# register the Python function +# 注册 Python 函数 table_env.register_function("hash_code", udf(HashCode(), DataTypes.BIGINT(), DataTypes.BIGINT())) -# use the function in Python Table API +# 在 Python Table API 中使用函数 my_table.select("string, bigint, string.hash_code(), hash_code(string)") -# use the function in SQL API +# 在 SQL API 中使用函数 table_env.sql_query("SELECT string, bigint, hash_code(bigint) FROM MyTable") {% endhighlight %} -There are many ways to define a Python scalar function besides extending the base class `ScalarFunction`. -Please refer to the [Python Scalar Function]({{ site.baseurl }}/zh/dev/table/python/python_udfs.html#scalar-functions) documentation for more details. +除了继承 `ScalarFunction`，还有很多方法可以定义 Python 标量函数。 +更多细节，可以参考 [Python 标量函数]({{ site.baseurl }}/zh/dev/table/python/python_udfs.html#scalar-functions) 文档。
{% top %} -Table Functions +表值函数 --------------- -Similar to a user-defined scalar function, a user-defined table function takes zero, one, or multiple scalar values as input parameters. However in contrast to a scalar function, it can return an arbitrary number of rows as output instead of a single value. The returned rows may consist of one or more columns. +跟自定义标量函数一样，自定义表值函数的输入参数也可以是 0 到多个标量。但是跟标量函数只能返回一个值不同的是，它可以返回任意多行。返回的每一行可以包含 1 到多列。
-In order to define a table function one has to extend the base class `TableFunction` in `org.apache.flink.table.functions` and implement (one or more) evaluation methods. The behavior of a table function is determined by its evaluation methods. An evaluation method must be declared `public` and named `eval`. The `TableFunction` can be overloaded by implementing multiple methods named `eval`. The parameter types of the evaluation methods determine all valid parameters of the table function. Evaluation methods can also support variable arguments, such as `eval(String... strs)`. The type of the returned table is determined by the generic type of `TableFunction`. Evaluation methods emit output rows using the protected `collect(T)` method. +要定义一个表值函数，你需要扩展 `org.apache.flink.table.functions` 下的 `TableFunction`，并且实现（一个或者多个）求值方法。表值函数的行为取决于你实现的求值方法。求值方法必须被声明为 `public`，并且名字必须是 `eval`。可以通过实现多个名为 `eval` 的方法对求值方法进行重载。求值方法的参数类型决定了表值函数的参数类型。表值函数也可以支持变长参数，比如 `eval(String... strs)`。表值函数返回的表的类型取决于 `TableFunction` 的泛型参数。求值方法通过 `collect(T)` 方法来发送要输出的行。 -In the Table API, a table function is used with `.joinLateral` or `.leftOuterJoinLateral`. The `joinLateral` operator (cross) joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator). The `leftOuterJoinLateral` operator joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator) and preserves outer rows for which the table function returns an empty table. In SQL use `LATERAL TABLE()` with CROSS JOIN and LEFT JOIN with an ON TRUE join condition (see examples below). +在 Table API 中，表值函数是通过 `.joinLateral` 或者 `.leftOuterJoinLateral` 来使用的。`joinLateral` 算子会把外表（算子左侧的表）的每一行跟跟表值函数返回的所有行（位于算子右侧）进行 （cross）join。`leftOuterJoinLateral` 算子也是把外表（算子左侧的表）的每一行跟表值函数返回的所有行（位于算子右侧）进行（cross）join，并且如果表值函数返回 0 行也会保留外表的这一行。在 SQL 里面用 CORSS JOIN 或者 以 ON TRUE 为条件的 LEFT JOIN 来配合 `LATERAL TABLE（)` 的使用。 -The following example shows how to define table-valued function, register it in the TableEnvironment, and call it in a query. Note that you can configure your table function via a constructor before it is registered: +下面的例子展示了如何定义一个表值函数，如何在 TableEnvironment 中注册表值函数，以及如何在查询中使用表值函数。你可以在注册之前通过构造函数来配置你的表值函数： {% highlight java %} -// The generic type "Tuple2" determines the schema of the returned table as (String, Integer). +// 泛型参数的类型 "Tuple2" 决定了返回的表的 schema 是（String，Integer）。 public class Split extends TableFunction> { private String separator = " "; @@ -190,7 +188,7 @@ public class Split extends TableFunction> { public void eval(String str) { for (String s : str.split(separator)) { - // use collect(...) to emit a row + // 使用 collect(...) 来输出一行数据 collect(new Tuple2(s, s.length())); } } @@ -199,36 +197,61 @@ public class Split extends TableFunction> { BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env); Table myTable = ... // table schema: [a: String] -// Register the function. +// 注册表值函数。 tableEnv.registerFunction("split", new Split("#")); -// Use the table function in the Java Table API. "as" specifies the field names of the table. +// 在 Java Table API 中使用表值函数。"as" 指明了表的字段名字 myTable.joinLateral("split(a) as (word, length)") .select("a, word, length"); myTable.leftOuterJoinLateral("split(a) as (word, length)") .select("a, word, length"); -// Use the table function in SQL with LATERAL and TABLE keywords. -// CROSS JOIN a table function (equivalent to "join" in Table API). +// 在 SQL 中用 LATERAL 和 TABLE 关键字来使用表值函数 +// CROSS JOIN a table function (等价于 Table API 中的 "join"). tableEnv.sqlQuery("SELECT a, word, length FROM MyTable, LATERAL TABLE(split(a)) as T(word, length)"); -// LEFT JOIN a table function (equivalent to "leftOuterJoin" in Table API). +// LEFT JOIN a table function (等价于 in Table API 中的 "leftOuterJoin"). tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(split(a)) as T(word, length) ON TRUE"); {% endhighlight %} + +需要注意的是 POJO 类型没有确定的字段顺序。所以，你不可以用 `AS` 来重命名返回的 POJO 的字段。 + +`TableFunction` 的返回类型默认是用 Flink 自动类型推导来决定的。对于基础类型和简单的 POJO 类型推导是没有问题的，但是对于更复杂的、自定义的、以及组合的类型可能会推导错误。如果有这种情况，可以通过重写（override） `TableFunction#getResultType()` 并且返回 `TypeInformation` 来指定返回类型。 + +下面的例子展示了 `TableFunction` 返回了一个 `Row` 类型，需要显示指定返回类型。我们通过重写 `TableFunction#getResultType` 来指定 `RowTypeInfo(String, Integer)` 作为返回的表的类型。 + +{% highlight java %} +public class CustomTypeSplit extends TableFunction { + public void eval(String str) { + for (String s : str.split(" ")) { + Row row = new Row(2); + row.setField(0, s); + row.setField(1, s.length()); + collect(row); + } + } + + @Override + public TypeInformation getResultType() { + return Types.ROW(Types.STRING(), Types.INT()); + } +} +{% endhighlight %} +
-In order to define a table function one has to extend the base class `TableFunction` in `org.apache.flink.table.functions` and implement (one or more) evaluation methods. The behavior of a table function is determined by its evaluation methods. An evaluation method must be declared `public` and named `eval`. The `TableFunction` can be overloaded by implementing multiple methods named `eval`. The parameter types of the evaluation methods determine all valid parameters of the table function. Evaluation methods can also support variable arguments, such as `eval(String... strs)`. The type of the returned table is determined by the generic type of `TableFunction`. Evaluation methods emit output rows using the protected `collect(T)` method. +要定义一个表值函数，你需要扩展 `org.apache.flink.table.functions` 下的 `TableFunction`，并且实现（一个或者多个）求值方法。表值函数的行为取决于你实现的求值方法。求值方法必须被声明为 `public`，并且名字必须是 `eval`。可以通过实现多个名为 `eval` 的方法对求值方法进行重载。求值方法的参数类型决定了表值函数的参数类型。表值函数也可以支持变长参数，比如 `eval(String... strs)`。表值函数返回的表的类型取决于 `TableFunction` 的泛型参数。求值方法通过 `collect(T)` 方法来发送要输出的行。 -In the Table API, a table function is used with `.joinLateral` or `.leftOuterJoinLateral`. The `joinLateral` operator (cross) joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator). The `leftOuterJoinLateral` operator joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator) and preserves outer rows for which the table function returns an empty table. In SQL use `LATERAL TABLE()` with CROSS JOIN and LEFT JOIN with an ON TRUE join condition (see examples below). +在 Table API 中，表值函数是通过 `.joinLateral` 或者 `.leftOuterJoinLateral` 来使用的。`joinLateral` 算子会把外表（算子左侧的表）的每一行跟跟表值函数返回的所有行（位于算子右侧）进行 （cross）join。`leftOuterJoinLateral` 算子也是把外表（算子左侧的表）的每一行跟表值函数返回的所有行（位于算子右侧）进行（cross）join，并且如果表值函数返回 0 行也会保留外表的这一行。在 SQL 里面用 CORSS JOIN 或者 以 ON TRUE 为条件的 LEFT JOIN 来配合 `LATERAL TABLE（)` 的使用。 -The following example shows how to define table-valued function, register it in the TableEnvironment, and call it in a query. Note that you can configure your table function via a constructor before it is registered: +下面的例子展示了如何定义一个表值函数，如何在 TableEnvironment 中注册表值函数，以及如何在查询中使用表值函数。你可以在注册之前通过构造函数来配置你的表值函数： {% highlight scala %} -// The generic type "(String, Int)" determines the schema of the returned table as (String, Integer). +// 泛型参数的类型 "(String, Int)" 决定了返回的表的 schema 是 (String, Integer)。 class Split(separator: String) extends TableFunction[(String, Int)] { def eval(str: String): Unit = { - // use collect(...) to emit a row. + // 使用 collect(...) 来输出一行 str.split(separator).foreach(x => collect((x, x.length))) } } @@ -236,30 +259,53 @@ class Split(separator: String) extends TableFunction[(String, Int)] { val tableEnv = BatchTableEnvironment.create(env) val myTable = ... // table schema: [a: String] -// Use the table function in the Scala Table API (Note: No registration required in Scala Table API). +// 在 Scala Table API 中使用表值函数（注意：在 Scala Table API 中不需要注册函数） val split = new Split("#") -// "as" specifies the field names of the generated table. +// "as" 指明了返回表的字段名字 myTable.joinLateral(split('a) as ('word, 'length)).select('a, 'word, 'length) myTable.leftOuterJoinLateral(split('a) as ('word, 'length)).select('a, 'word, 'length) -// Register the table function to use it in SQL queries. +// 注册表值函数，然后才能在 SQL 查询中使用 tableEnv.registerFunction("split", new Split("#")) -// Use the table function in SQL with LATERAL and TABLE keywords. +// 在 SQL 中使用 LATERAL 和 TABLE 关键字类使用表值函数 // CROSS JOIN a table function (equivalent to "join" in Table API) tableEnv.sqlQuery("SELECT a, word, length FROM MyTable, LATERAL TABLE(split(a)) as T(word, length)") // LEFT JOIN a table function (equivalent to "leftOuterJoin" in Table API) tableEnv.sqlQuery("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(split(a)) as T(word, length) ON TRUE") {% endhighlight %} -**IMPORTANT:** Do not implement TableFunction as a Scala object. Scala object is a singleton and will cause concurrency issues. +**重要：**不要把表值函数实现成一个 Scala object。Scala object 是一个单例，会有并发的问题。 + +需要注意的是 POJO 类型没有确定的字段顺序。所以，你不可以用 `AS` 来重命名返回的 POJO 的字段。 + +`TableFunction` 的返回类型默认是用 Flink 自动类型推导来决定的。对于基础类型和简单的 POJO 类型推导是没有问题的，但是对于更复杂的、自定义的、以及组合的类型可能会推导错误。如果有这种情况，可以通过重写（override） `TableFunction#getResultType()` 并且返回 `TypeInformation` 来指定返回类型。 + +下面的例子展示了 `TableFunction` 返回了一个 `Row` 类型，需要显示指定返回类型。我们通过重写 `TableFunction#getResultType` 来返回 `RowTypeInfo` 作为返回类型。 + +{% highlight scala %} +class CustomTypeSplit extends TableFunction[Row] { + def eval(str: String): Unit = { + str.split(" ").foreach({ s => + val row = new Row(2) + row.setField(0, s) + row.setField(1, s.length) + collect(row) + }) + } + + override def getResultType: TypeInformation[Row] = { + Types.ROW(Types.STRING, Types.INT) + } +} +{% endhighlight %}
-In order to define a Python table function, one can extend the base class `TableFunction` in `pyflink.table.udtf` and Implement an evaluation method. The behavior of a Python table function is determined by the evaluation method which is named eval. +要实现一个 Python 表值函数，你可以扩展 `pyflink.table.udtf` 下的 `TableFunction`，并且实现一个求值方法。Python 表值函数的行为取决于你实现的求值方法，它的名字必须是 `eval`。 -In the Python Table API, a Python table function is used with `.join_lateral` or `.left_outer_join_lateral`. The `join_lateral` operator (cross) joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator). The `left_outer_join_lateral` operator joins each row from the outer table (table on the left of the operator) with all rows produced by the table-valued function (which is on the right side of the operator) and preserves outer rows for which the table function returns an empty table. In SQL use `LATERAL TABLE()` with CROSS JOIN and LEFT JOIN with an ON TRUE join condition (see examples below). +在 Python Table API 中，表值函数是通过 `.join_lateral` 或者 `.left_outer_join_lateral` 来使用的。`join_lateral` 算子会把外表（算子左侧的表）的每一行跟跟表值函数返回的所有行（位于算子右侧）进行 （cross）join。`left_outer_join_lateral` 算子也是把外表（算子左侧的表）的每一行跟表值函数返回的所有行（位于算子右侧）进行（cross）join，并且如果表值函数返回 0 行也会保留外表的这一行。在 SQL 里面用 CORSS JOIN 或者 以 ON TRUE 为条件的 LEFT JOIN 来配合 `LATERAL TABLE（)` 的使用。 -The following example shows how to define a Python table function, registered it in the TableEnvironment, and call it in a query. Note that you can configure your table function via a constructor before it is registered: +下面的例子展示了如何定义一个 Python 表值函数，如何在 TableEnvironment 中注册表值函数，以及如何在查询中使用表值函数。你可以在注册之前通过构造函数来配置你的表值函数： {% highlight python %} class Split(TableFunction): @@ -271,115 +317,60 @@ env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(env) my_table = ... # type: Table, table schema: [a: String] -# register the Python Table Function +# 注册 Python 表值函数 table_env.register_function("split", udtf(Split(), DataTypes.STRING(), [DataTypes.STRING(), DataTypes.INT()])) -# use the Python Table Function in Python Table API +# 在 Python Table API 中使用 Python 表值函数 my_table.join_lateral("split(a) as (word, length)") my_table.left_outer_join_lateral("split(a) as (word, length)") -# use the Python Table function in SQL API +# 在 SQL API 中使用 Python 表值函数 table_env.sql_query("SELECT a, word, length FROM MyTable, LATERAL TABLE(split(a)) as T(word, length)") table_env.sql_query("SELECT a, word, length FROM MyTable LEFT JOIN LATERAL TABLE(split(a)) as T(word, length) ON TRUE") {% endhighlight %} -There are many ways to define a Python table function besides extending the base class `TableFunction`. -Please refer to the [Python Table Function]({{ site.baseurl }}/zh/dev/table/python/python_udfs.html#table-functions) documentation for more details. - -
-
- -
-
-Please note that POJO types do not have a deterministic field order. Therefore, you cannot rename the fields of POJO returned by a table function using `AS`. - -By default the result type of a `TableFunction` is determined by Flink’s automatic type extraction facilities. This works well for basic types and simple POJOs but might be wrong for more complex, custom, or composite types. In such a case, the type of the result can be manually specified by overriding `TableFunction#getResultType()` which returns its `TypeInformation`. - -The following example shows an example of a `TableFunction` that returns a `Row` type which requires explicit type information. We define that the returned table type should be `RowTypeInfo(String, Integer)` by overriding `TableFunction#getResultType()`. - -{% highlight java %} -public class CustomTypeSplit extends TableFunction { - public void eval(String str) { - for (String s : str.split(" ")) { - Row row = new Row(2); - row.setField(0, s); - row.setField(1, s.length()); - collect(row); - } - } - - @Override - public TypeInformation getResultType() { - return Types.ROW(Types.STRING(), Types.INT()); - } -} -{% endhighlight %} -
- -
-Please note that POJO types do not have a deterministic field order. Therefore, you cannot rename the fields of POJO returned by a table function using `AS`. - -By default the result type of a `TableFunction` is determined by Flink’s automatic type extraction facilities. This works well for basic types and simple POJOs but might be wrong for more complex, custom, or composite types. In such a case, the type of the result can be manually specified by overriding `TableFunction#getResultType()` which returns its `TypeInformation`. - -The following example shows an example of a `TableFunction` that returns a `Row` type which requires explicit type information. We define that the returned table type should be `RowTypeInfo(String, Integer)` by overriding `TableFunction#getResultType()`. - -{% highlight scala %} -class CustomTypeSplit extends TableFunction[Row] { - def eval(str: String): Unit = { - str.split(" ").foreach({ s => - val row = new Row(2) - row.setField(0, s) - row.setField(1, s.length) - collect(row) - }) - } +除了继承 `TableFunction`，还有很多其它方法可以定义 Python 表值函数。 +更多信息，参考 [Python 表值函数]({{ site.baseurl }}/zh/dev/table/python/python_udfs.html#table-functions)文档。 - override def getResultType: TypeInformation[Row] = { - Types.ROW(Types.STRING, Types.INT) - } -} -{% endhighlight %}
{% top %} -Aggregation Functions +聚合函数 --------------------- -User-Defined Aggregate Functions (UDAGGs) aggregate a table (one or more rows with one or more attributes) to a scalar value. +自定义聚合函数（UDAGG）是把一个表（一行或者多行，每行可以有一列或者多列）聚合成一个标量值。
UDAGG mechanism
-The above figure shows an example of an aggregation. Assume you have a table that contains data about beverages. The table consists of three columns, `id`, `name` and `price` and 5 rows. Imagine you need to find the highest price of all beverages in the table, i.e., perform a `max()` aggregation. You would need to check each of the 5 rows and the result would be a single numeric value. +上面的图片展示了一个聚合的例子。假设你有一个关于饮料的表。表里面有三个字段，分别是 `id`、`name`、`price`，表里有 5 行数据。假设你需要找到所有饮料里最贵的饮料的价格，即执行一个 `max()` 聚合。你需要遍历所有 5 行数据，而结果就只有一个数值。 -User-defined aggregation functions are implemented by extending the `AggregateFunction` class. An `AggregateFunction` works as follows. First, it needs an `accumulator`, which is the data structure that holds the intermediate result of the aggregation. An empty accumulator is created by calling the `createAccumulator()` method of the `AggregateFunction`. Subsequently, the `accumulate()` method of the function is called for each input row to update the accumulator. Once all rows have been processed, the `getValue()` method of the function is called to compute and return the final result. +自定义聚合函数是通过扩展 `AggregateFunction` 来实现的。`AggregateFunction` 的工作过程如下。首先，它需要一个 `accumulator`，它是一个数据结构，存储了聚合的中间结果。通过调用 `AggregateFunction` 的 `createAccumulator()` 方法创建一个空的 accumulator。接下来，对于每一行数据，会调用 `accumulate()` 方法来更新 accumulator。当所有的数据都处理完了之后，通过调用 `getValue` 方法来计算和返回最终的结果。 -**The following methods are mandatory for each `AggregateFunction`:** +**下面几个方法是每个 `AggregateFunction` 必须要实现的：** - `createAccumulator()` - `accumulate()` - `getValue()` -Flink’s type extraction facilities can fail to identify complex data types, e.g., if they are not basic types or simple POJOs. So similar to `ScalarFunction` and `TableFunction`, `AggregateFunction` provides methods to specify the `TypeInformation` of the result type (through - `AggregateFunction#getResultType()`) and the type of the accumulator (through `AggregateFunction#getAccumulatorType()`). +Flink 的类型推导在遇到复杂类型的时候可能会推导出错误的结果，比如那些非基本类型和普通的 POJO 类型的复杂类型。所以跟 `ScalarFunction` 和 `TableFunction` 一样，`AggregateFunction` 也提供了 `AggregateFunction#getResultType()` 和 `AggregateFunction#getAccumulatorType()` 来分别指定返回值类型和 accumulator 的类型，两个函数的返回值类型也都是 `TypeInformation`。 -Besides the above methods, there are a few contracted methods that can be -optionally implemented. While some of these methods allow the system more efficient query execution, others are mandatory for certain use cases. For instance, the `merge()` method is mandatory if the aggregation function should be applied in the context of a session group window (the accumulators of two session windows need to be joined when a row is observed that "connects" them). +除了上面的方法，还有几个方法可以选择实现。这些方法有些可以让查询更加高效，而有些是在某些特定场景下必须要实现的。例如，如果聚合函数用在会话窗口（当两个会话窗口合并的时候需要 merge 他们的 accumulator）的话，`merge()` 方法就是必须要实现的。 -**The following methods of `AggregateFunction` are required depending on the use case:** +**`AggregateFunction` 的以下方法在某些场景下是必须实现的：** -- `retract()` is required for aggregations on bounded `OVER` windows. -- `merge()` is required for many batch aggregations and session window aggregations. -- `resetAccumulator()` is required for many batch aggregations. +- `retract()` 在 bounded `OVER` 窗口中是必须实现的。 +- `merge()` 在许多批式聚合和会话窗口聚合中是必须实现的。 +- `resetAccumulator()` 在许多批式聚合中是必须实现的。 -All methods of `AggregateFunction` must be declared as `public`, not `static` and named exactly as the names mentioned above. The methods `createAccumulator`, `getValue`, `getResultType`, and `getAccumulatorType` are defined in the `AggregateFunction` abstract class, while others are contracted methods. In order to define a aggregate function, one has to extend the base class `org.apache.flink.table.functions.AggregateFunction` and implement one (or more) `accumulate` methods. The method `accumulate` can be overloaded with different parameter types and supports variable arguments. +`AggregateFunction` 的所有方法都必须是 `public` 的，不能是 `static` 的，而且名字必须跟上面写的一样。`createAccumulator`、`getValue`、`getResultType` 以及 `getAccumulatorType` 这几个函数是在抽象类 `AggregateFunction` 中定义的，而其他函数都是约定的方法。如果要定义一个聚合函数，你需要扩展 `org.apache.flink.table.functions.AggregateFunction`，并且实现一个（或者多个）`accumulate` 方法。`accumulate` 方法可以重载，每个方法的参数类型不同，并且支持变长参数。 -Detailed documentation for all methods of `AggregateFunction` is given below. +`AggregateFunction` 的所有方法的详细文档如下。
@@ -603,15 +594,15 @@ abstract class AggregateFunction[T, ACC] extends UserDefinedAggregateFunction[T,
-The following example shows how to +下面的例子展示了如何： -- define an `AggregateFunction` that calculates the weighted average on a given column, -- register the function in the `TableEnvironment`, and -- use the function in a query. +- 定义一个聚合函数来计算某一列的加权平均， +- 在 `TableEnvironment` 中注册函数， +- 在查询中使用函数。 -To calculate an weighted average value, the accumulator needs to store the weighted sum and count of all the data that has been accumulated. In our example we define a class `WeightedAvgAccum` to be the accumulator. Accumulators are automatically backup-ed by Flink's checkpointing mechanism and restored in case of a failure to ensure exactly-once semantics. +为了计算加权平均值，accumulator 需要存储加权总和以及数据的条数。在我们的例子里，我们定义了一个类 `WeightedAvgAccum` 来作为 accumulator。Flink 的 checkpoint 机制会自动保存 accumulator，在失败时进行恢复，以此来保证精确一次的语义。 -The `accumulate()` method of our `WeightedAvg` `AggregateFunction` has three inputs. The first one is the `WeightedAvgAccum` accumulator, the other two are user-defined inputs: input value `ivalue` and weight of the input `iweight`. Although the `retract()`, `merge()`, and `resetAccumulator()` methods are not mandatory for most aggregation types, we provide them below as examples. Please note that we used Java primitive types and defined `getResultType()` and `getAccumulatorType()` methods in the Scala example because Flink type extraction does not work very well for Scala types. +我们的 `WeightedAvg`（聚合函数）的 `accumulate` 方法有三个输入参数。第一个是 `WeightedAvgAccum` accumulator，另外两个是用户自定义的输入：输入的值 `ivalue` 和 输入的权重 `iweight`。尽管 `retract()`、`merge()`、`resetAccumulator()` 这几个方法在大多数聚合类型中都不是必须实现的，我们也在样例中提供了他们的实现。请注意我们在 Scala 样例中也是用的是 Java 的基础类型，并且定义了 `getResultType()` 和 `getAccumulatorType()`，因为 Flink 的类型推导对于 Scala 的类型推导做的不是很好。
@@ -668,11 +659,11 @@ public static class WeightedAvg extends AggregateFunction UDAGG mechanism -The above figure shows an example of a table aggregation. Assume you have a table that contains data about beverages. The table consists of three columns, `id`, `name` and `price` and 5 rows. Imagine you need to find the top 2 highest prices of all beverages in the table, i.e., perform a `top2()` table aggregation. You would need to check each of the 5 rows and the result would be a table with the top 2 values. +上图展示了一个表值聚合函数的例子。假设你有一个饮料的表，这个表有 3 列，分别是 `id`、`name` 和 `price`，一共有 5 行。假设你需要找到价格最高的两个饮料，类似于 `top2()` 表值聚合函数。你需要遍历所有 5 行数据，结果是有 2 行数据的一个表。 -User-defined table aggregation functions are implemented by extending the `TableAggregateFunction` class. A `TableAggregateFunction` works as follows. First, it needs an `accumulator`, which is the data structure that holds the intermediate result of the aggregation. An empty accumulator is created by calling the `createAccumulator()` method of the `TableAggregateFunction`. Subsequently, the `accumulate()` method of the function is called for each input row to update the accumulator. Once all rows have been processed, the `emitValue()` method of the function is called to compute and return the final results. +用户自定义表值聚合函数是通过扩展 `TableAggregateFunction` 类来实现的。一个 `TableAggregateFunction` 的工作过程如下。首先，它需要一个 `accumulator`，这个 `accumulator` 负责存储聚合的中间结果。 通过调用 `TableAggregateFunction` 的 `createAccumulator` 方法来构造一个空的 accumulator。接下来，对于每一行数据，会调用 `accumulate` 方法来更新 accumulator。当所有数据都处理完之后，调用 `emitValue` 方法来计算和返回最终的结果。 -**The following methods are mandatory for each `TableAggregateFunction`:** +**下面几个 `TableAggregateFunction` 的方法是必须要实现的：** - `createAccumulator()` - `accumulate()` -Flink’s type extraction facilities can fail to identify complex data types, e.g., if they are not basic types or simple POJOs. So similar to `ScalarFunction` and `TableFunction`, `TableAggregateFunction` provides methods to specify the `TypeInformation` of the result type (through - `TableAggregateFunction#getResultType()`) and the type of the accumulator (through `TableAggregateFunction#getAccumulatorType()`). +Flink 的类型推导在遇到复杂类型的时候可能会推导出错误的结果，比如那些非基本类型和普通的 POJO 类型的复杂类型。所以类似于 `ScalarFunction` 和 `TableFunction`，`TableAggregateFunction` 也提供了 `TableAggregateFunction#getResultType()` 和 `TableAggregateFunction#getAccumulatorType()` 方法来指定返回值类型和 accumulator 的类型，这两个方法都需要返回 `TypeInformation`。 -Besides the above methods, there are a few contracted methods that can be -optionally implemented. While some of these methods allow the system more efficient query execution, others are mandatory for certain use cases. For instance, the `merge()` method is mandatory if the aggregation function should be applied in the context of a session group window (the accumulators of two session windows need to be joined when a row is observed that "connects" them). +除了上面的方法，还有几个其他的方法可以选择性的实现。有些方法可以让查询更加高效，而有些方法对于某些特定场景是必须要实现的。比如，在会话窗口（当两个会话窗口合并时会合并两个 accumulator）中使用聚合函数时，必须要实现`merge()` 方法。 -**The following methods of `TableAggregateFunction` are required depending on the use case:** +**下面几个 `TableAggregateFunction` 的方法在某些特定场景下是必须要实现的：** -- `retract()` is required for aggregations on bounded `OVER` windows. -- `merge()` is required for many batch aggregations and session window aggregations. -- `resetAccumulator()` is required for many batch aggregations. -- `emitValue()` is required for batch and window aggregations. +- `retract()` 在 bounded `OVER` 窗口中的聚合函数必须要实现。 +- `merge()` 在许多批式聚合和会话窗口聚合中是必须要实现的。 +- `resetAccumulator()` 在许多批式聚合中是必须要实现的。 +- `emitValue()` 在批式聚合以及窗口聚合中是必须要实现的。 -**The following methods of `TableAggregateFunction` are used to improve the performance of streaming jobs:** +**下面的 `TableAggregateFunction` 的方法可以提升流式任务的效率：** -- `emitUpdateWithRetract()` is used to emit values that have been updated under retract mode. +- `emitUpdateWithRetract()` 在 retract 模式下，该方法负责发送被更新的值。 -For `emitValue` method, it emits full data according to the accumulator. Take TopN as an example, `emitValue` emit all top n values each time. This may bring performance problems for streaming jobs. To improve the performance, a user can also implement `emitUpdateWithRetract` method to improve the performance. The method outputs data incrementally in retract mode, i.e., once there is an update, we have to retract old records before sending new updated ones. The method will be used in preference to the `emitValue` method if they are all defined in the table aggregate function, because `emitUpdateWithRetract` is treated to be more efficient than `emitValue` as it can output values incrementally. +`emitValue` 方法会发送所有 accumulator 给出的结果。拿 TopN 来说，`emitValue` 每次都会发送所有的最大的 n 个值。这在流式任务中可能会有一些性能问题。为了提升性能，用户可以实现 `emitUpdateWithRetract` 方法。这个方法在 retract 模式下会增量的输出结果，比如有数据更新了，我们必须要撤回老的数据，然后再发送新的数据。如果定义了 `emitUpdateWithRetract` 方法，那它会优先于 `emitValue` 方法被使用，因为一般认为 `emitUpdateWithRetract` 会更加高效，因为它的输出是增量的。 -All methods of `TableAggregateFunction` must be declared as `public`, not `static` and named exactly as the names mentioned above. The methods `createAccumulator`, `getResultType`, and `getAccumulatorType` are defined in the parent abstract class of `TableAggregateFunction`, while others are contracted methods. In order to define a table aggregate function, one has to extend the base class `org.apache.flink.table.functions.TableAggregateFunction` and implement one (or more) `accumulate` methods. The method `accumulate` can be overloaded with different parameter types and supports variable arguments. +`TableAggregateFunction` 的所有方法都必须是 `public` 的、非 `static` 的，而且名字必须跟上面提到的一样。`createAccumulator`、`getResultType` 和 `getAccumulatorType` 这三个方法是在抽象父类 `TableAggregateFunction` 中定义的，而其他的方法都是约定的方法。要实现一个表值聚合函数，你必须扩展 `org.apache.flink.table.functions.TableAggregateFunction`，并且实现一个（或者多个）`accumulate` 方法。`accumulate` 方法可以有多个重载的方法，也可以支持变长参数。 -Detailed documentation for all methods of `TableAggregateFunction` is given below. +`TableAggregateFunction` 的所有方法的详细文档如下。
@@ -1126,15 +1116,15 @@ abstract class TableAggregateFunction[T, ACC] extends UserDefinedAggregateFuncti
-The following example shows how to +下面的例子展示了如何 -- define a `TableAggregateFunction` that calculates the top 2 values on a given column, -- register the function in the `TableEnvironment`, and -- use the function in a Table API query(TableAggregateFunction is only supported by Table API). +- 定义一个 `TableAggregateFunction` 来计算给定列的最大的 2 个值， +- 在 `TableEnvironment` 中注册函数， +- 在 Table API 查询中使用函数（当前只在 Table API 中支持 TableAggregateFunction）。 -To calculate the top 2 values, the accumulator needs to store the biggest 2 values of all the data that has been accumulated. In our example we define a class `Top2Accum` to be the accumulator. Accumulators are automatically backup-ed by Flink's checkpointing mechanism and restored in case of a failure to ensure exactly-once semantics. +为了计算最大的 2 个值，accumulator 需要保存当前看到的最大的 2 个值。在我们的例子中，我们定义了类 `Top2Accum` 来作为 accumulator。Flink 的 checkpoint 机制会自动保存 accumulator，并且在失败时进行恢复，来保证精确一次的语义。 -The `accumulate()` method of our `Top2` `TableAggregateFunction` has two inputs. The first one is the `Top2Accum` accumulator, the other one is the user-defined input: input value `v`. Although the `merge()` method is not mandatory for most table aggregation types, we provide it below as examples. Please note that we used Java primitive types and defined `getResultType()` and `getAccumulatorType()` methods in the Scala example because Flink type extraction does not work very well for Scala types. +我们的 `Top2` 表值聚合函数（`TableAggregateFunction`）的 `accumulate()` 方法有两个输入，第一个是 `Top2Accum` accumulator，另一个是用户定义的输入：输入的值 `v`。尽管 `merge()` 方法在大多数聚合类型中不是必须的，我们也在样例中提供了它的实现。请注意，我们在 Scala 样例中也使用的是 Java 的基础类型，并且定义了 `getResultType()` 和 `getAccumulatorType()` 方法，因为 Flink 的类型推导对于 Scala 的类型推导支持的不是很好。
@@ -1188,14 +1178,14 @@ public static class Top2 extends TableAggregateFunction } } -// register function +// 注册函数 StreamTableEnvironment tEnv = ... tEnv.registerFunction("top2", new Top2()); -// init table +// 初始化表 Table tab = ...; -// use function +// 使用函数 tab.groupBy("key") .flatAggregate("top2(a) as (v, rank)") .select("key, v, rank"); @@ -1258,10 +1248,10 @@ class Top2 extends TableAggregateFunction[JTuple2[JInteger, JInteger], Top2Accum } } -// init table +// 初始化表 val tab = ... -// use function +// 使用函数 tab .groupBy('key) .flatAggregate(top2('a) as ('v, 'rank)) @@ -1272,7 +1262,7 @@ tab
-The following example shows how to use `emitUpdateWithRetract` method to emit only updates. To emit only updates, in our example, the accumulator keeps both old and new top 2 values. Note: if the N of topN is big, it may inefficient to keep both old and new values. One way to solve this case is to store the input record into the accumulator in `accumulate` method and then perform calculation in `emitUpdateWithRetract`. +下面的例子展示了如何使用 `emitUpdateWithRetract` 方法来只发送更新的数据。为了只发送更新的结果，accumulator 保存了上一次的最大的2个值，也保存了当前最大的2个值。注意：如果 TopN 中的 n 非常大，这种既保存上次的结果，也保存当前的结果的方式不太高效。一种解决这种问题的方式是把输入数据直接存储到 `accumulator` 中，然后在调用 `emitUpdateWithRetract` 方法时再进行计算。
@@ -1332,14 +1322,14 @@ public static class Top2 extends TableAggregateFunction } } -// register function +// 注册函数 StreamTableEnvironment tEnv = ... tEnv.registerFunction("top2", new Top2()); -// init table +// 初始化表 Table tab = ...; -// use function +// 使用函数 tab.groupBy("key") .flatAggregate("top2(a) as (v, rank)") .select("key, v, rank"); @@ -1409,10 +1399,10 @@ class Top2 extends TableAggregateFunction[JTuple2[JInteger, JInteger], Top2Accum } } -// init table +// 初始化表 val tab = ... -// use function +// 使用函数 tab .groupBy('key) .flatAggregate(top2('a) as ('v, 'rank)) @@ -1425,33 +1415,33 @@ tab {% top %} -Best Practices for Implementing UDFs +实现自定义函数的最佳实践 ------------------------------------ -The Table API and SQL code generation internally tries to work with primitive values as much as possible. A user-defined function can introduce much overhead through object creation, casting, and (un)boxing. Therefore, it is highly recommended to declare parameters and result types as primitive types instead of their boxed classes. `Types.DATE` and `Types.TIME` can also be represented as `int`. `Types.TIMESTAMP` can be represented as `long`. +在 Table API 和 SQL 的内部，代码生成会尽量的使用基础类型。自定义函数的参数及返回值类型是对象，会有很多的对象创建、转换（cast）、以及自动拆装箱的开销。因此，强烈建议使用基础类型来作为参数以及返回值的类型。`Types.DATE` 和 `Types.TIME` 可以用 `int` 来表示。`Types.TIMESTAMP` 可以用 `long` 来表示。 -We recommended that user-defined functions should be written by Java instead of Scala as Scala types pose a challenge for Flink's type extractor. +我们建议自定义函数用 Java 来实现，而不是用 Scala 来实现，因为 Flink 的类型推导对 Scala 不是很友好。 {% top %} -Integrating UDFs with the Runtime +自定义函数跟运行时集成 --------------------------------- -Sometimes it might be necessary for a user-defined function to get global runtime information or do some setup/clean-up work before the actual work. User-defined functions provide `open()` and `close()` methods that can be overridden and provide similar functionality as the methods in `RichFunction` of DataSet or DataStream API. +有时候自定义函数需要获取一些全局信息，或者在真正被调用之前做一些配置（setup）/清理（clean-up）的工作。自定义函数也提供了 `open()` 和 `close()` 方法，你可以重写这两个方法做到类似于 DataSet 或者 DataStream API 中 `RichFunction` 的功能。 -The `open()` method is called once before the evaluation method. The `close()` method after the last call to the evaluation method. +`open()` 方法在求值方法被调用之前先调用。`close()` 方法在求值方法调用完之后被调用。 -The `open()` method provides a `FunctionContext` that contains information about the context in which user-defined functions are executed, such as the metric group, the distributed cache files, or the global job parameters. +`open()` 方法提供了一个 `FunctionContext`，它包含了一些自定义函数被执行时的上下文信息，比如 metric group、分布式文件缓存，或者是全局的作业参数等。 -The following information can be obtained by calling the corresponding methods of `FunctionContext`: +下面的信息可以通过调用 `FunctionContext` 的对应的方法来获得： -| Method | Description | +| 方法 | 描述 | | :------------------------------------ | :----------------------------------------------------- | -| `getMetricGroup()` | Metric group for this parallel subtask. | -| `getCachedFile(name)` | Local temporary file copy of a distributed cache file. | -| `getJobParameter(name, defaultValue)` | Global job parameter value associated with given key. | +| `getMetricGroup()` | 执行该函数的 subtask 的 Metric Group。 | +| `getCachedFile(name)` | 分布式文件缓存的本地临时文件副本。 | +| `getJobParameter(name, defaultValue)` | 跟对应的 key 关联的全局参数值。 | -The following example snippet shows how to use `FunctionContext` in a scalar function for accessing a global job parameter: +下面的例子展示了如何在一个标量函数中通过 `FunctionContext` 来获取一个全局的任务参数：
@@ -1462,8 +1452,8 @@ public class HashCode extends ScalarFunction { @Override public void open(FunctionContext context) throws Exception { - // access "hashcode_factor" parameter - // "12" would be the default value if parameter does not exist + // 获取参数 "hashcode_factor" + // 如果不存在，则使用默认值 "12" factor = Integer.valueOf(context.getJobParameter("hashcode_factor", "12")); } @@ -1475,18 +1465,18 @@ public class HashCode extends ScalarFunction { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env); -// set job parameter +// 设置任务参数 Configuration conf = new Configuration(); conf.setString("hashcode_factor", "31"); env.getConfig().setGlobalJobParameters(conf); -// register the function +// 注册函数 tableEnv.registerFunction("hashCode", new HashCode()); -// use the function in Java Table API +// 在 Java Table API 中使用函数 myTable.select("string, string.hashCode(), hashCode(string)"); -// use the function in SQL +// 在 SQL 中使用函数 tableEnv.sqlQuery("SELECT string, HASHCODE(string) FROM MyTable"); {% endhighlight %}
@@ -1498,8 +1488,8 @@ object hashCode extends ScalarFunction { var hashcode_factor = 12 override def open(context: FunctionContext): Unit = { - // access "hashcode_factor" parameter - // "12" would be the default value if parameter does not exist + // 获取参数 "hashcode_factor" + // 如果不存在，则使用默认值 "12" hashcode_factor = context.getJobParameter("hashcode_factor", "12").toInt } @@ -1510,10 +1500,10 @@ object hashCode extends ScalarFunction { val tableEnv = BatchTableEnvironment.create(env) -// use the function in Scala Table API +// 在 Scala Table API 中使用函数 myTable.select('string, hashCode('string)) -// register and use the function in SQL +// 在 SQL 中注册和使用函数 tableEnv.registerFunction("hashCode", hashCode) tableEnv.sqlQuery("SELECT string, HASHCODE(string) FROM MyTable") {% endhighlight %} From 90982ca1a9595bdadfb3433fbfdbabe6097e254d Mon Sep 17 00:00:00 2001 From: Rui Li Date: Tue, 19 May 2020 15:01:41 +0800 Subject: [PATCH 028/773] [FLINK-17786][sql-client] Fix can not switch dialect in SQL CLI Remove dialect from ExecutionEntry This closes #12217 --- .../client/config/entries/ExecutionEntry.java | 15 ---------- .../gateway/local/ExecutionContext.java | 1 - .../gateway/local/ExecutionContextTest.java | 17 ----------- .../gateway/local/LocalExecutorITCase.java | 29 +++++++++++++++++++ .../resources/test-sql-client-dialect.yaml | 6 +--- 5 files changed, 30 insertions(+), 38 deletions(-) diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/ExecutionEntry.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/ExecutionEntry.java index dd7c3222ce536..78be7f52e0765 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/ExecutionEntry.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/ExecutionEntry.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.SqlDialect; import org.apache.flink.table.client.config.ConfigUtil; import org.apache.flink.table.client.config.Environment; import org.apache.flink.table.descriptors.DescriptorProperties; @@ -111,8 +110,6 @@ public class ExecutionEntry extends ConfigEntry { public static final String EXECUTION_CURRENT_DATABASE = "current-database"; - public static final String EXECUTION_SQL_DIALECT = "dialect"; - private ExecutionEntry(DescriptorProperties properties) { super(properties); } @@ -157,12 +154,6 @@ protected void validate(DescriptorProperties properties) { properties.validateInt(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL, true, 1); properties.validateString(EXECUTION_CURRENT_CATALOG, true, 1); properties.validateString(EXECUTION_CURRENT_DATABASE, true, 1); - properties.validateEnumValues(EXECUTION_SQL_DIALECT, - true, - Arrays.asList( - SqlDialect.DEFAULT.name().toLowerCase(), - SqlDialect.HIVE.name().toLowerCase() - )); } public EnvironmentSettings getEnvironmentSettings() { @@ -339,12 +330,6 @@ public boolean isTableauMode() { .orElse(false); } - public SqlDialect getSqlDialect() { - return properties.getOptionalString(EXECUTION_SQL_DIALECT) - .map(name -> SqlDialect.valueOf(name.toUpperCase())) - .orElse(SqlDialect.DEFAULT); - } - public Map asTopLevelMap() { return properties.asPrefixedMap(EXECUTION_ENTRY + '.'); } diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java index 6a5b9963546a4..4396ba96b9d63 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java @@ -456,7 +456,6 @@ private void initializeTableEnvironment(@Nullable SessionState sessionState) { config.addConfiguration(flinkConfig); environment.getConfiguration().asMap().forEach((k, v) -> config.getConfiguration().setString(k, v)); - config.setSqlDialect(environment.getExecution().getSqlDialect()); if (noInheritedState) { //-------------------------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java index b68ebefb91133..f44513e8787d8 100644 --- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java +++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java @@ -25,7 +25,6 @@ import org.apache.flink.client.python.PythonFunctionFactory; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; -import org.apache.flink.table.api.SqlDialect; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.config.ExecutionConfigOptions; @@ -74,7 +73,6 @@ public class ExecutionContextTest { public static final String CATALOGS_ENVIRONMENT_FILE = "test-sql-client-catalogs.yaml"; private static final String STREAMING_ENVIRONMENT_FILE = "test-sql-client-streaming.yaml"; private static final String CONFIGURATION_ENVIRONMENT_FILE = "test-sql-client-configuration.yaml"; - private static final String DIALECT_ENVIRONMENT_FILE = "test-sql-client-dialect.yaml"; private static final String FUNCTION_ENVIRONMENT_FILE = "test-sql-client-python-functions.yaml"; @Test @@ -307,21 +305,6 @@ public void testInitCatalogs() throws Exception{ Collections.singletonList(new DefaultCLI(flinkConfig))).build(); } - @Test - public void testSQLDialect() throws Exception { - ExecutionContext context = createDefaultExecutionContext(); - assertEquals(SqlDialect.DEFAULT, context.getTableEnvironment().getConfig().getSqlDialect()); - - Map replaceVars = new HashMap<>(); - replaceVars.put("$VAR_DIALECT", "default"); - context = createExecutionContext(DIALECT_ENVIRONMENT_FILE, replaceVars); - assertEquals(SqlDialect.DEFAULT, context.getTableEnvironment().getConfig().getSqlDialect()); - - replaceVars.put("$VAR_DIALECT", "hive"); - context = createExecutionContext(DIALECT_ENVIRONMENT_FILE, replaceVars); - assertEquals(SqlDialect.HIVE, context.getTableEnvironment().getConfig().getSqlDialect()); - } - @SuppressWarnings("unchecked") private ExecutionContext createExecutionContext(String file, Map replaceVars) throws Exception { final Environment env = EnvironmentFileUtil.parseModified( diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java index cb96387f86ef1..f7b56d44d6556 100644 --- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java +++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java @@ -32,8 +32,10 @@ import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.configuration.WebOptions; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.table.api.SqlDialect; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.config.OptimizerConfigOptions; +import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.table.catalog.hive.HiveCatalog; import org.apache.flink.table.client.config.Environment; import org.apache.flink.table.client.config.entries.ExecutionEntry; @@ -107,6 +109,7 @@ public static List planner() { } private static final String DEFAULTS_ENVIRONMENT_FILE = "test-sql-client-defaults.yaml"; + private static final String DIALECT_ENVIRONMENT_FILE = "test-sql-client-dialect.yaml"; private static final int NUM_TMS = 2; private static final int NUM_SLOTS_PER_TM = 2; @@ -1304,6 +1307,32 @@ public void testAlterFunction() throws Exception { } } + @Test + public void testSQLDialect() throws Exception { + LocalExecutor executor = createDefaultExecutor(clusterClient); + final SessionContext session = new SessionContext("test-session", new Environment()); + String sessionId = executor.openSession(session); + // by default to use DEFAULT dialect + assertEquals(SqlDialect.DEFAULT, executor.getExecutionContext(sessionId).getTableEnvironment().getConfig().getSqlDialect()); + // test switching dialect + executor.setSessionProperty(sessionId, TableConfigOptions.TABLE_SQL_DIALECT.key(), "hive"); + assertEquals(SqlDialect.HIVE, executor.getExecutionContext(sessionId).getTableEnvironment().getConfig().getSqlDialect()); + executor.closeSession(sessionId); + + Map replaceVars = new HashMap<>(); + replaceVars.put("$VAR_DIALECT", "default"); + executor = createModifiedExecutor(DIALECT_ENVIRONMENT_FILE, clusterClient, replaceVars); + sessionId = executor.openSession(session); + assertEquals(SqlDialect.DEFAULT, executor.getExecutionContext(sessionId).getTableEnvironment().getConfig().getSqlDialect()); + executor.closeSession(sessionId); + + replaceVars.put("$VAR_DIALECT", "hive"); + executor = createModifiedExecutor(DIALECT_ENVIRONMENT_FILE, clusterClient, replaceVars); + sessionId = executor.openSession(session); + assertEquals(SqlDialect.HIVE, executor.getExecutionContext(sessionId).getTableEnvironment().getConfig().getSqlDialect()); + executor.closeSession(sessionId); + } + private void executeStreamQueryTable( Map replaceVars, String query, diff --git a/flink-table/flink-sql-client/src/test/resources/test-sql-client-dialect.yaml b/flink-table/flink-sql-client/src/test/resources/test-sql-client-dialect.yaml index d4fa4bf3ba7bf..a60be03a1543d 100644 --- a/flink-table/flink-sql-client/src/test/resources/test-sql-client-dialect.yaml +++ b/flink-table/flink-sql-client/src/test/resources/test-sql-client-dialect.yaml @@ -25,10 +25,6 @@ execution: planner: blink type: batch result-mode: table - dialect: "$VAR_DIALECT" configuration: - table.exec.sort.default-limit: 100 - table.exec.spill-compression.enabled: true - table.exec.spill-compression.block-size: 128kb - table.optimizer.join-reorder-enabled: true + table.sql-dialect: "$VAR_DIALECT" From fdec46d267ca938522f170a38fd8eb268941aa03 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 18 May 2020 15:23:05 +0200 Subject: [PATCH 029/773] [FLINK-17790][kafka] Fix JDK 11 compile error --- .../streaming/connectors/kafka/table/KafkaOptions.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaOptions.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaOptions.java index 3e326fd1bdb69..337fe76eb714f 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaOptions.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/table/KafkaOptions.java @@ -263,7 +263,6 @@ public static Properties getKafkaProperties(Map tableOptions) { /** * The partitioner can be either "fixed", "round-robin" or a customized partitioner full class name. */ - @SuppressWarnings({"unchecked", "rawtypes"}) public static Optional> getFlinkKafkaPartitioner( ReadableConfig tableOptions, ClassLoader classLoader) { @@ -340,8 +339,7 @@ private static boolean hasKafkaClientProperties(Map tableOptions /** * Returns a class value with the given class name. */ - @SuppressWarnings("rawtypes") - private static FlinkKafkaPartitioner initializePartitioner(String name, ClassLoader classLoader) { + private static FlinkKafkaPartitioner initializePartitioner(String name, ClassLoader classLoader) { try { Class clazz = Class.forName(name, true, classLoader); if (!FlinkKafkaPartitioner.class.isAssignableFrom(clazz)) { @@ -350,7 +348,10 @@ private static FlinkKafkaPartitioner initializePartitioner(String name, ClassLoa name, FlinkKafkaPartitioner.class.getName())); } - return InstantiationUtil.instantiate(name, FlinkKafkaPartitioner.class, classLoader); + @SuppressWarnings("unchecked") + final FlinkKafkaPartitioner kafkaPartitioner = InstantiationUtil.instantiate(name, FlinkKafkaPartitioner.class, classLoader); + + return kafkaPartitioner; } catch (ClassNotFoundException | FlinkException e) { throw new ValidationException( String.format("Could not find and instantiate partitioner class '%s'", name), e); From 2f1a6b0aa3eeae973e8a6124bef888f223bf43ae Mon Sep 17 00:00:00 2001 From: PengFei Li Date: Thu, 14 May 2020 10:40:33 +0800 Subject: [PATCH 030/773] [FLINK-16076][docs-zh] Translate "Queryable State" page into Chinese This closes #12139. --- docs/dev/stream/state/queryable_state.zh.md | 197 ++++++++------------ 1 file changed, 75 insertions(+), 122 deletions(-) diff --git a/docs/dev/stream/state/queryable_state.zh.md b/docs/dev/stream/state/queryable_state.zh.md index 3c14c7e8360a9..1d62efda375be 100644 --- a/docs/dev/stream/state/queryable_state.zh.md +++ b/docs/dev/stream/state/queryable_state.zh.md @@ -1,5 +1,5 @@ --- -title: "可查询状态" +title: "Queryable State" nav-parent_id: streaming_state nav-pos: 4 is_beta: true @@ -27,75 +27,52 @@ under the License. {:toc}
- Note: The client APIs for queryable state are currently in an evolving state and - there are no guarantees made about stability of the provided interfaces. It is - likely that there will be breaking API changes on the client side in the upcoming Flink versions. + 注意: 目前 querable state 的客户端 API 还在不断演进，不保证现有接口的稳定性。在后续的 Flink 版本中有可能发生 API 变化。
-In a nutshell, this feature exposes Flink's managed keyed (partitioned) state -(see [Working with State]({{ site.baseurl }}/dev/stream/state/state.html)) to the outside world and -allows the user to query a job's state from outside Flink. For some scenarios, queryable state -eliminates the need for distributed operations/transactions with external systems such as key-value -stores which are often the bottleneck in practice. In addition, this feature may be particularly -useful for debugging purposes. +简而言之, 这个特性将 Flink 的 managed keyed (partitioned) state +(参考 [Working with State]({{ site.baseurl }}/zh/dev/stream/state/state.html)) 暴露给外部，从而用户可以在 Flink 外部查询作业 state。 +在某些场景中，Queryable State 消除了对外部系统的分布式操作以及事务的需求，比如 KV 存储系统，而这些外部系统往往会成为瓶颈。除此之外，这个特性对于调试作业非常有用。
- Attention: When querying a state object, that object is accessed from a concurrent - thread without any synchronization or copying. This is a design choice, as any of the above would lead - to increased job latency, which we wanted to avoid. Since any state backend using Java heap space, - e.g. MemoryStateBackend or FsStateBackend, does not work - with copies when retrieving values but instead directly references the stored values, read-modify-write - patterns are unsafe and may cause the queryable state server to fail due to concurrent modifications. - The RocksDBStateBackend is safe from these issues. + 注意: 进行查询时，state 会在并发线程中被访问，但 state 不会进行同步和拷贝。这种设计是为了避免同步和拷贝带来的作业延时。对于使用 Java 堆内存的 state backend， + 比如 MemoryStateBackend 或者 FsStateBackend，它们获取状态时不会进行拷贝，而是直接引用状态对象，所以对状态的 read-modify-write 是不安全的，并且可能会因为并发修改导致查询失败。但 RocksDBStateBackend 是安全的，不会遇到上述问题。
-## Architecture +## 架构 -Before showing how to use the Queryable State, it is useful to briefly describe the entities that compose it. -The Queryable State feature consists of three main entities: +在展示如何使用 Queryable State 之前，先简单描述一下该特性的组成部分，主要包括以下三部分: - 1. the `QueryableStateClient`, which (potentially) runs outside the Flink cluster and submits the user queries, - 2. the `QueryableStateClientProxy`, which runs on each `TaskManager` (*i.e.* inside the Flink cluster) and is responsible - for receiving the client's queries, fetching the requested state from the responsible Task Manager on his behalf, and - returning it to the client, and - 3. the `QueryableStateServer` which runs on each `TaskManager` and is responsible for serving the locally stored state. + 1. `QueryableStateClient`，默认运行在 Flink 集群外部，负责提交用户的查询请求； + 2. `QueryableStateClientProxy`，运行在每个 `TaskManager` 上(*即* Flink 集群内部)，负责接收客户端的查询请求，从所负责的 Task Manager 获取请求的 state，并返回给客户端； + 3. `QueryableStateServer`, 运行在 `TaskManager` 上，负责服务本地存储的 state。 -The client connects to one of the proxies and sends a request for the state associated with a specific -key, `k`. As stated in [Working with State]({{ site.baseurl }}/dev/stream/state/state.html), keyed state is organized in -*Key Groups*, and each `TaskManager` is assigned a number of these key groups. To discover which `TaskManager` is -responsible for the key group holding `k`, the proxy will ask the `JobManager`. Based on the answer, the proxy will -then query the `QueryableStateServer` running on that `TaskManager` for the state associated with `k`, and forward the -response back to the client. +客户端连接到一个代理，并发送请求获取特定 `k` 对应的 state。 如 [Working with State]({{ site.baseurl }}/zh/dev/stream/state/state.html) 所述，keyed state 按照 +*Key Groups* 进行划分，每个 `TaskManager` 会分配其中的一些 key groups。代理会询问 `JobManager` 以找到 `k` 所属 key group 的 TaskManager。根据返回的结果, 代理将会向运行在 `TaskManager` 上的 `QueryableStateServer` 查询 `k` 对应的 state， 并将结果返回给客户端。 -## Activating Queryable State +## 激活 Queryable State -To enable queryable state on your Flink cluster, you need to do the following: +为了在 Flink 集群上使用 queryable state，需要进行以下操作： - 1. copy the `flink-queryable-state-runtime{{ site.scala_version_suffix }}-{{site.version }}.jar` -from the `opt/` folder of your [Flink distribution](https://flink.apache.org/downloads.html "Apache Flink: Downloads"), -to the `lib/` folder. - 2. set the property `queryable-state.enable` to `true`. See the [Configuration]({{ site.baseurl }}/ops/config.html#queryable-state) documentation for details and additional parameters. + 1. 将 `flink-queryable-state-runtime{{ site.scala_version_suffix }}-{{site.version }}.jar` +从 [Flink distribution](https://flink.apache.org/downloads.html "Apache Flink: Downloads") 的 `opt/` 目录拷贝到 `lib/` 目录； + 2. 将参数 `queryable-state.enable` 设置为 `true`。详细信息以及其它配置可参考文档 [Configuration]({{ site.baseurl }}/zh/ops/config.html#queryable-state)。 -To verify that your cluster is running with queryable state enabled, check the logs of any -task manager for the line: `"Started the Queryable State Proxy Server @ ..."`. +为了验证集群的 queryable state 已经被激活，可以检查任意 task manager 的日志中是否包含 "Started the Queryable State Proxy Server @ ..."。 -## Making State Queryable +## 将 state 设置为可查询的 -Now that you have activated queryable state on your cluster, it is time to see how to use it. In order for a state to -be visible to the outside world, it needs to be explicitly made queryable by using: +激活集群的 queryable state 功能后，还要将 state 设置为可查询的才能对外可见，可以通过以下两种方式进行设置： -* either a `QueryableStateStream`, a convenience object which acts as a sink and offers its incoming values as queryable -state, or -* the `stateDescriptor.setQueryable(String queryableStateName)` method, which makes the keyed state represented by the - state descriptor, queryable. +* 创建 `QueryableStateStream`，它会作为一个 sink，并将输入数据转化为 queryable state； +* 通过 `stateDescriptor.setQueryable(String queryableStateName)` 将 state 描述符所表示的 keyed state 设置成可查询的。 -The following sections explain the use of these two approaches. +接下来的部分将详细解释这两种方式。 ### Queryable State Stream -Calling `.asQueryableState(stateName, stateDescriptor)` on a `KeyedStream` returns a `QueryableStateStream` which offers -its values as queryable state. Depending on the type of state, there are the following variants of the `asQueryableState()` -method: +在 `KeyedStream` 上调用 `.asQueryableState(stateName, stateDescriptor)` 将会返回一个 `QueryableStateStream`， 它会将流数据转化为 queryable state。 +对应不同的 state 类型，`asQueryableState()` 有以下一些方法变体： {% highlight java %} // ValueState @@ -119,28 +96,23 @@ QueryableStateStream asQueryableState(
- Note: There is no queryable ListState sink as it would result in an ever-growing - list which may not be cleaned up and thus will eventually consume too much memory. + 注意: 没有可查询的 ListState sink，因为这种情况下 list 会不断增长，并且可能不会被清理，最终会消耗大量的内存。
-The returned `QueryableStateStream` can be seen as a sink and **cannot** be further transformed. Internally, a -`QueryableStateStream` gets translated to an operator which uses all incoming records to update the queryable state -instance. The updating logic is implied by the type of the `StateDescriptor` provided in the `asQueryableState` call. -In a program like the following, all records of the keyed stream will be used to update the state instance via the -`ValueState.update(value)`: +返回的 `QueryableStateStream` 可以被视作一个sink，而且**不能再**被进一步转换。在内部实现上，一个 `QueryableStateStream` 被转换成一个 operator，使用输入的数据来更新 queryable state。state 如何更新是由 `asQueryableState` 提供的 `StateDescriptor` 来决定的。在下面的代码中, keyed stream 的所有数据将会通过 `ValueState.update(value)` 来更新状态： {% highlight java %} stream.keyBy(0).asQueryableState("query-name") {% endhighlight %} -This acts like the Scala API's `flatMapWithState`. +这个行为类似于 Scala API 中的 `flatMapWithState`。 ### Managed Keyed State -Managed keyed state of an operator -(see [Using Managed Keyed State]({{ site.baseurl }}/dev/stream/state/state.html#using-managed-keyed-state)) -can be made queryable by making the appropriate state descriptor queryable via -`StateDescriptor.setQueryable(String queryableStateName)`, as in the example below: +operator 中的 Managed keyed state +(参考 [Using Managed Keyed State]({{ site.baseurl }}/zh/dev/stream/state/state.html#using-managed-keyed-state)) +可以通过 `StateDescriptor.setQueryable(String queryableStateName)` 将 state descriptor 设置成可查询的，从而使 state 可查询，如下所示： + {% highlight java %} ValueStateDescriptor> descriptor = new ValueStateDescriptor<>( @@ -150,20 +122,17 @@ descriptor.setQueryable("query-name"); // queryable state name {% endhighlight %}
- Note: The queryableStateName parameter may be chosen arbitrarily and is only - used for queries. It does not have to be identical to the state's own name. + 注意: 参数 queryableStateName 可以任意选取，并且只被用来进行查询，它可以和 state 的名称不同。
-This variant has no limitations as to which type of state can be made queryable. This means that this can be used for -any `ValueState`, `ReduceState`, `ListState`, `MapState`, `AggregatingState`, and the currently deprecated `FoldingState`. +这种方式不会限制 state 类型，即任意的 `ValueState`、`ReduceState`、`ListState`、`MapState`、`AggregatingState` 以及已弃用的 `FoldingState` +均可作为 queryable state。 -## Querying State +## 查询 state -So far, you have set up your cluster to run with queryable state and you have declared (some of) your state as -queryable. Now it is time to see how to query this state. +目前为止，你已经激活了集群的 queryable state 功能，并且将一些 state 设置成了可查询的，接下来将会展示如何进行查询。 -For this you can use the `QueryableStateClient` helper class. This is available in the `flink-queryable-state-client` -jar which must be explicitly included as a dependency in the `pom.xml` of your project along with `flink-core`, as shown below: +为了进行查询，可以使用辅助类 `QueryableStateClient`，这个类位于 `flink-queryable-state-client` 的 jar 中，在项目的 `pom.xml` 需要显示添加对 `flink-queryable-state-client` 和 `flink-core` 的依赖, 如下所示：
{% highlight xml %} @@ -180,18 +149,16 @@ jar which must be explicitly included as a dependency in the `pom.xml` of your p {% endhighlight %}
-For more on this, you can check how to [set up a Flink program]({{ site.baseurl }}/dev/projectsetup/dependencies.html). +关于依赖的更多信息, 可以参考如何 [配置 Flink 项目]({{ site.baseurl }}/zh/dev/projectsetup/dependencies.html). -The `QueryableStateClient` will submit your query to the internal proxy, which will then process your query and return -the final result. The only requirement to initialize the client is to provide a valid `TaskManager` hostname (remember -that there is a queryable state proxy running on each task manager) and the port where the proxy listens. More on how -to configure the proxy and state server port(s) in the [Configuration Section](#configuration). +`QueryableStateClient` 将提交你的请求到内部代理，代理会处理请求并返回结果。客户端的初始化只需要提供一个有效的 `TaskManager` 主机名 +(每个 task manager 上都运行着一个 queryable state 代理)，以及代理监听的端口号。关于如何配置代理以及端口号可以参考 [Configuration Section](#configuration). {% highlight java %} QueryableStateClient client = new QueryableStateClient(tmHostname, proxyPort); {% endhighlight %} -With the client ready, to query a state of type `V`, associated with a key of type `K`, you can use the method: +客户端就绪后，为了查询类型为 `K` 的 key，以及类型为 `V` 的state，可以使用如下方法： {% highlight java %} CompletableFuture getKvState( @@ -202,35 +169,29 @@ CompletableFuture getKvState( StateDescriptor stateDescriptor) {% endhighlight %} -The above returns a `CompletableFuture` eventually holding the state value for the queryable state instance identified -by `queryableStateName` of the job with ID `jobID`. The `key` is the key whose state you are interested in and the -`keyTypeInfo` will tell Flink how to serialize/deserialize it. Finally, the `stateDescriptor` contains the necessary -information about the requested state, namely its type (`Value`, `Reduce`, etc) and the necessary information on how -to serialize/deserialize it. +该方法会返回一个最终将包含 state 的 queryable state 实例，该实例可通过 JobID 和 queryableStateName 识别。在方法参数中，`key` 用来指定所要查询的状态所属的 key。 +`keyTypeInfo` 告诉 Flink 如何对 key 进行序列化和反序列化。`stateDescriptor` 包含了所请求 state 的必要信息，即 state 类型(`Value`，`Reduce` 等等)， +以及如何对其进行序列化和反序列。 -The careful reader will notice that the returned future contains a value of type `S`, *i.e.* a `State` object containing -the actual value. This can be any of the state types supported by Flink: `ValueState`, `ReduceState`, `ListState`, `MapState`, -`AggregatingState`, and the currently deprecated `FoldingState`. +细心的读者会注意到返回的 future 包含类型为 `S` 的值，*即*一个存储实际值的 `State` 对象。它可以是Flink支持的任何类型的 state：`ValueState`、`ReduceState`、 +`ListState`、`MapState`、`AggregatingState` 以及弃用的 `FoldingState`。
- Note: These state objects do not allow modifications to the contained state. You can use them to get - the actual value of the state, e.g. using valueState.get(), or iterate over - the contained entries, e.g. using the mapState.entries(), but you cannot - modify them. As an example, calling the add() method on a returned list state will throw an - UnsupportedOperationException. + 注意： 这些 state 对象不允许对其中的 state 进行修改。你可以通过 valueState.get() 获取实际的 state， + 或者通过 mapState.entries() 遍历所有 ，但是不能修改它们。举例来说，对返回的 list state 调用 add() + 方法将会导致 UnsupportedOperationException。
- Note: The client is asynchronous and can be shared by multiple threads. It needs - to be shutdown via QueryableStateClient.shutdown() when unused in order to free - resources. + 注意: 客户端是异步的，并且可能被多个线程共享。客户端不再使用后需要通过 QueryableStateClient.shutdown() + 来终止，从而释放资源。
-### Example +### 示例 -The following example extends the `CountWindowAverage` example -(see [Using Managed Keyed State]({{ site.baseurl }}/dev/stream/state/state.html#using-managed-keyed-state)) -by making it queryable and shows how to query this value: +下面的例子扩展自 `CountWindowAverage` +(参考 [Using Managed Keyed State]({{ site.baseurl }}/zh/dev/stream/state/state.html#using-managed-keyed-state))， +将其中的 state 设置成可查询的，并展示了如何进行查询： {% highlight java %} public class CountWindowAverage extends RichFlatMapFunction, Tuple2> { @@ -262,7 +223,7 @@ public class CountWindowAverage extends RichFlatMapFunction, } {% endhighlight %} -Once used in a job, you can retrieve the job ID and then query any key's current state from this operator: +上面的代码作为作业运行后，可以获取作业的 ID，然后可以通过下面的方式查询任何 key 下的 state。 {% highlight java %} QueryableStateClient client = new QueryableStateClient(tmHostname, proxyPort); @@ -288,34 +249,26 @@ resultFuture.thenAccept(response -> { ## Configuration -The following configuration parameters influence the behaviour of the queryable state server and client. -They are defined in `QueryableStateOptions`. +下面的配置会影响 queryable state 服务器端和客户端的行为，它们定义在 `QueryableStateOptions`。 ### State Server -* `queryable-state.server.ports`: the server port range of the queryable state server. This is useful to avoid port clashes if more - than 1 task managers run on the same machine. The specified range can be: a port: "9123", a range of ports: "50100-50200", - or a list of ranges and or points: "50100-50200,50300-50400,51234". The default port is 9067. -* `queryable-state.server.network-threads`: number of network (event loop) threads receiving incoming requests for the state server (0 => #slots) -* `queryable-state.server.query-threads`: number of threads handling/serving incoming requests for the state server (0 => #slots). +* `queryable-state.server.ports`: 服务器端口范围，如果同一台机器上运行了多个 task manager，可以避免端口冲突。指定的可以是一个具体的端口号，如 "9123"， + 可以是一个端口范围，如 "50100-50200"，或者可以是端口范围以及端口号的组合，如 "50100-50200,50300-50400,51234"。默认端口号是 9067。 +* `queryable-state.server.network-threads`: 服务器端 network (event loop) thread 的数量，用来接收查询请求 (如果设置为0，则线程数为 slot 数)。 +* `queryable-state.server.query-threads`: 服务器端处理查询请求的线程数 (如果设置为0，则线程数为 slot 数)。 ### Proxy -* `queryable-state.proxy.ports`: the server port range of the queryable state proxy. This is useful to avoid port clashes if more - than 1 task managers run on the same machine. The specified range can be: a port: "9123", a range of ports: "50100-50200", - or a list of ranges and or points: "50100-50200,50300-50400,51234". The default port is 9069. -* `queryable-state.proxy.network-threads`: number of network (event loop) threads receiving incoming requests for the client proxy (0 => #slots) -* `queryable-state.proxy.query-threads`: number of threads handling/serving incoming requests for the client proxy (0 => #slots). - -## Limitations - -* The queryable state life-cycle is bound to the life-cycle of the job, *e.g.* tasks register -queryable state on startup and unregister it on disposal. In future versions, it is desirable to -decouple this in order to allow queries after a task finishes, and to speed up recovery via state -replication. -* Notifications about available KvState happen via a simple tell. In the future this should be improved to be -more robust with asks and acknowledgements. -* The server and client keep track of statistics for queries. These are currently disabled by -default as they would not be exposed anywhere. As soon as there is better support to publish these -numbers via the Metrics system, we should enable the stats. +* `queryable-state.proxy.ports`: 代理的服务端口范围。如果同一台机器上运行了多个 task manager，可以避免端口冲突。指定的可以是一个具体的端口号，如 "9123"， + 可以是一个端口范围，如"50100-50200"，或者可以是端口范围以及端口号的组合，如 "50100-50200,50300-50400,51234"。默认端口号是 9069。 +* `queryable-state.proxy.network-threads`: 代理上 network (event loop) thread 的数量，用来接收查询请求 (如果设置为0，则线程数为 slot 数)。 +* `queryable-state.proxy.query-threads`: 代理上处理查询请求的线程数 (如果设置为0，则线程数为 slot 数)。 + +## 限制 + +* queryable state 的生命周期受限于作业的生命周期，*比如* tasks 在启动时注册可查询状态，并在退出时注销。在后续版本中，希望能够将其解耦 +从而允许 task 结束后依然能够查询 state，并且通过 state 备份来加速恢复。 +* 目前是通过 tell 来通知可用的 KvState。将来会使用 asks 和 acknowledgements 来提升稳定性。 +* 服务器端和客户端会记录请求的统计信息。因为统计信息目前不会暴露给外部，所以这个功能默认没有开启。如果将来支持通过 Metrics 系统发布这些数据，将开启统计功能。 {% top %} From 8078c86c546dc879151eedcbd4c156bfc4e304b0 Mon Sep 17 00:00:00 2001 From: yangyichao-mango <1048262223@qq.com> Date: Sun, 17 May 2020 15:04:28 +0800 Subject: [PATCH 031/773] [FLINK-17353][docs] Fix Broken links in Flink docs master This closes #12196 --- docs/concepts/flink-architecture.zh.md | 132 ++++++++++ docs/dev/connectors/elasticsearch.md | 2 +- docs/dev/connectors/elasticsearch.zh.md | 2 +- docs/dev/stream/state/checkpointing.md | 4 +- docs/dev/stream/state/checkpointing.zh.md | 4 +- docs/dev/table/common.zh.md | 2 +- docs/dev/user_defined_functions.zh.md | 241 ++++++++++++++++++ .../flink-operations-playground.md | 4 +- .../flink-operations-playground.zh.md | 4 +- .../walkthroughs/python_table_api.zh.md | 2 +- docs/index.md | 7 +- docs/index.zh.md | 5 +- docs/internals/task_lifecycle.md | 2 +- docs/internals/task_lifecycle.zh.md | 2 +- docs/monitoring/metrics.zh.md | 2 +- docs/ops/config.md | 2 +- docs/ops/config.zh.md | 2 +- docs/ops/memory/mem_migration.zh.md | 6 +- docs/ops/memory/mem_trouble.zh.md | 4 +- docs/ops/memory/mem_tuning.zh.md | 3 +- docs/ops/python_shell.zh.md | 2 +- docs/ops/state/savepoints.md | 2 +- docs/ops/state/savepoints.zh.md | 2 +- 23 files changed, 406 insertions(+), 32 deletions(-) create mode 100644 docs/concepts/flink-architecture.zh.md create mode 100644 docs/dev/user_defined_functions.zh.md diff --git a/docs/concepts/flink-architecture.zh.md b/docs/concepts/flink-architecture.zh.md new file mode 100644 index 0000000000000..8414943fd167d --- /dev/null +++ b/docs/concepts/flink-architecture.zh.md @@ -0,0 +1,132 @@ +--- +title: Flink Architecture +nav-id: flink-architecture +nav-pos: 4 +nav-title: Flink Architecture +nav-parent_id: concepts +--- + + +* This will be replaced by the TOC +{:toc} + +## Flink Applications and Flink Sessions + +`TODO: expand this section` + +{% top %} + +## Anatomy of a Flink Cluster + +`TODO: expand this section, especially about components of the Flink Master and +container environments` + +The Flink runtime consists of two types of processes: + + - The *Flink Master* coordinates the distributed execution. It schedules + tasks, coordinates checkpoints, coordinates recovery on failures, etc. + + There is always at least one *Flink Master*. A high-availability setup + might have multiple *Flink Masters*, one of which is always the + *leader*, and the others are *standby*. + + - The *TaskManagers* (also called *workers*) execute the *tasks* (or more + specifically, the subtasks) of a dataflow, and buffer and exchange the data + *streams*. + + There must always be at least one TaskManager. + +The Flink Master and TaskManagers can be started in various ways: directly on +the machines as a [standalone cluster]({% link +ops/deployment/cluster_setup.md %}), in containers, or managed by resource +frameworks like [YARN]({% link ops/deployment/yarn_setup.md +%}) or [Mesos]({% link ops/deployment/mesos.md %}). +TaskManagers connect to Flink Masters, announcing themselves as available, and +are assigned work. + +The *client* is not part of the runtime and program execution, but is used to +prepare and send a dataflow to the Flink Master. After that, the client can +disconnect, or stay connected to receive progress reports. The client runs +either as part of the Java/Scala program that triggers the execution, or in the +command line process `./bin/flink run ...`. + +The processes involved in executing a Flink dataflow + +{% top %} + +## Tasks and Operator Chains + +For distributed execution, Flink *chains* operator subtasks together into +*tasks*. Each task is executed by one thread. Chaining operators together into +tasks is a useful optimization: it reduces the overhead of thread-to-thread +handover and buffering, and increases overall throughput while decreasing +latency. The chaining behavior can be configured; see the [chaining docs]({% +link dev/stream/operators/index.md %}#task-chaining-and-resource-groups) for +details. + +The sample dataflow in the figure below is executed with five subtasks, and +hence with five parallel threads. + +Operator chaining into Tasks + +{% top %} + +## Task Slots and Resources + +Each worker (TaskManager) is a *JVM process*, and may execute one or more +subtasks in separate threads. To control how many tasks a worker accepts, a +worker has so called **task slots** (at least one). + +Each *task slot* represents a fixed subset of resources of the TaskManager. A +TaskManager with three slots, for example, will dedicate 1/3 of its managed +memory to each slot. Slotting the resources means that a subtask will not +compete with subtasks from other jobs for managed memory, but instead has a +certain amount of reserved managed memory. Note that no CPU isolation happens +here; currently slots only separate the managed memory of tasks. + +By adjusting the number of task slots, users can define how subtasks are +isolated from each other. Having one slot per TaskManager means that each task +group runs in a separate JVM (which can be started in a separate container, for +example). Having multiple slots means more subtasks share the same JVM. Tasks +in the same JVM share TCP connections (via multiplexing) and heartbeat +messages. They may also share data sets and data structures, thus reducing the +per-task overhead. + +A TaskManager with Task Slots and Tasks + +By default, Flink allows subtasks to share slots even if they are subtasks of +different tasks, so long as they are from the same job. The result is that one +slot may hold an entire pipeline of the job. Allowing this *slot sharing* has +two main benefits: + + - A Flink cluster needs exactly as many task slots as the highest parallelism + used in the job. No need to calculate how many tasks (with varying + parallelism) a program contains in total. + + - It is easier to get better resource utilization. Without slot sharing, the + non-intensive *source/map()* subtasks would block as many resources as the + resource intensive *window* subtasks. With slot sharing, increasing the + base parallelism in our example from two to six yields full utilization of + the slotted resources, while making sure that the heavy subtasks are fairly + distributed among the TaskManagers. + +TaskManagers with shared Task Slots + +{% top %} diff --git a/docs/dev/connectors/elasticsearch.md b/docs/dev/connectors/elasticsearch.md index 5bc1404e3a4a6..4b8b2daa74f2b 100644 --- a/docs/dev/connectors/elasticsearch.md +++ b/docs/dev/connectors/elasticsearch.md @@ -317,7 +317,7 @@ time of checkpoints. This effectively assures that all requests before the checkpoint was triggered have been successfully acknowledged by Elasticsearch, before proceeding to process more records sent to the sink. -More details on checkpoints and fault tolerance are in the [fault tolerance docs]({{site.baseurl}}/internals/stream_checkpointing.html). +More details on checkpoints and fault tolerance are in the [fault tolerance docs]({{site.baseurl}}/training/fault_tolerance.html). To use fault tolerant Elasticsearch Sinks, checkpointing of the topology needs to be enabled at the execution environment: diff --git a/docs/dev/connectors/elasticsearch.zh.md b/docs/dev/connectors/elasticsearch.zh.md index 59219543d83f4..640f4d6144c17 100644 --- a/docs/dev/connectors/elasticsearch.zh.md +++ b/docs/dev/connectors/elasticsearch.zh.md @@ -317,7 +317,7 @@ time of checkpoints. This effectively assures that all requests before the checkpoint was triggered have been successfully acknowledged by Elasticsearch, before proceeding to process more records sent to the sink. -More details on checkpoints and fault tolerance are in the [fault tolerance docs]({{site.baseurl}}/internals/stream_checkpointing.html). +More details on checkpoints and fault tolerance are in the [fault tolerance docs]({{site.baseurl}}/zh/training/fault_tolerance.html). To use fault tolerant Elasticsearch Sinks, checkpointing of the topology needs to be enabled at the execution environment: diff --git a/docs/dev/stream/state/checkpointing.md b/docs/dev/stream/state/checkpointing.md index c193fc37d71ed..f5fef8949d01f 100644 --- a/docs/dev/stream/state/checkpointing.md +++ b/docs/dev/stream/state/checkpointing.md @@ -32,7 +32,7 @@ any type of more elaborate operation. In order to make state fault tolerant, Flink needs to **checkpoint** the state. Checkpoints allow Flink to recover state and positions in the streams to give the application the same semantics as a failure-free execution. -The [documentation on streaming fault tolerance]({{ site.baseurl }}/internals/stream_checkpointing.html) describes in detail the technique behind Flink's streaming fault tolerance mechanism. +The [documentation on streaming fault tolerance]({{ site.baseurl }}/training/fault_tolerance.html) describes in detail the technique behind Flink's streaming fault tolerance mechanism. ## Prerequisites @@ -173,7 +173,7 @@ Some more parameters and/or defaults may be set via `conf/flink-conf.yaml` (see ## Selecting a State Backend -Flink's [checkpointing mechanism]({{ site.baseurl }}/internals/stream_checkpointing.html) stores consistent snapshots +Flink's [checkpointing mechanism]({{ site.baseurl }}/training/fault_tolerance.html) stores consistent snapshots of all the state in timers and stateful operators, including connectors, windows, and any [user-defined state](state.html). Where the checkpoints are stored (e.g., JobManager memory, file system, database) depends on the configured **State Backend**. diff --git a/docs/dev/stream/state/checkpointing.zh.md b/docs/dev/stream/state/checkpointing.zh.md index d4aa989523a68..c940ada9de6d2 100644 --- a/docs/dev/stream/state/checkpointing.zh.md +++ b/docs/dev/stream/state/checkpointing.zh.md @@ -29,7 +29,7 @@ Flink 中的每个方法或算子都能够是**有状态的**（阅读 [working 状态化的方法在处理单个 元素/事件 的时候存储数据，让状态成为使各个类型的算子更加精细的重要部分。 为了让状态容错，Flink 需要为状态添加 **checkpoint（检查点）**。Checkpoint 使得 Flink 能够恢复状态和在流中的位置，从而向应用提供和无故障执行时一样的语义。 -[容错文档]({{ site.baseurl }}/zh/internals/stream_checkpointing.html) 中介绍了 Flink 流计算容错机制内部的技术原理。 +[容错文档]({{ site.baseurl }}/zh/training/fault_tolerance.html) 中介绍了 Flink 流计算容错机制内部的技术原理。 ## 前提条件 @@ -165,7 +165,7 @@ env.get_checkpoint_config().set_prefer_checkpoint_for_recovery(True) ## 选择一个 State Backend -Flink 的 [checkpointing 机制]({{ site.baseurl }}/zh/internals/stream_checkpointing.html) 会将 timer 以及 stateful 的 operator 进行快照，然后存储下来， +Flink 的 [checkpointing 机制]({{ site.baseurl }}/zh/training/fault_tolerance.html) 会将 timer 以及 stateful 的 operator 进行快照，然后存储下来， 包括连接器（connectors），窗口（windows）以及任何用户[自定义的状态](state.html)。 Checkpoint 存储在哪里取决于所配置的 **State Backend**（比如 JobManager memory、 file system、 database）。 diff --git a/docs/dev/table/common.zh.md b/docs/dev/table/common.zh.md index bd36b1e1af2d4..c10f2d371af34 100644 --- a/docs/dev/table/common.zh.md +++ b/docs/dev/table/common.zh.md @@ -561,7 +561,7 @@ revenue = orders \ Flink SQL 是基于实现了SQL标准的 [Apache Calcite](https://calcite.apache.org) 的。SQL 查询由常规字符串指定。 -文档 [SQL]({{ site.baseurl }}/zh/dev/table/sql.html) 描述了Flink对流处理和批处理表的SQL支持。 +文档 [SQL]({{ site.baseurl }}/zh/dev/table/sql/index.html) 描述了Flink对流处理和批处理表的SQL支持。 下面的示例演示了如何指定查询并将结果作为 `Table` 对象返回。 diff --git a/docs/dev/user_defined_functions.zh.md b/docs/dev/user_defined_functions.zh.md new file mode 100644 index 0000000000000..bdfbe54cee3c9 --- /dev/null +++ b/docs/dev/user_defined_functions.zh.md @@ -0,0 +1,241 @@ +--- +title: 'User-Defined Functions' +nav-id: user_defined_function +nav-parent_id: streaming +nav-pos: 4 +--- + + +Most operations require a user-defined function. This section lists different +ways of how they can be specified. We also cover `Accumulators`, which can be +used to gain insights into your Flink application. + +
+
+ +## Implementing an interface + +The most basic way is to implement one of the provided interfaces: + +{% highlight java %} +class MyMapFunction implements MapFunction { + public Integer map(String value) { return Integer.parseInt(value); } +}; +data.map(new MyMapFunction()); +{% endhighlight %} + +## Anonymous classes + +You can pass a function as an anonymous class: +{% highlight java %} +data.map(new MapFunction () { + public Integer map(String value) { return Integer.parseInt(value); } +}); +{% endhighlight %} + +## Java 8 Lambdas + +Flink also supports Java 8 Lambdas in the Java API. + +{% highlight java %} +data.filter(s -> s.startsWith("http://")); +{% endhighlight %} + +{% highlight java %} +data.reduce((i1,i2) -> i1 + i2); +{% endhighlight %} + +## Rich functions + +All transformations that require a user-defined function can +instead take as argument a *rich* function. For example, instead of + +{% highlight java %} +class MyMapFunction implements MapFunction { + public Integer map(String value) { return Integer.parseInt(value); } +}; +{% endhighlight %} + +you can write + +{% highlight java %} +class MyMapFunction extends RichMapFunction { + public Integer map(String value) { return Integer.parseInt(value); } +}; +{% endhighlight %} + +and pass the function as usual to a `map` transformation: + +{% highlight java %} +data.map(new MyMapFunction()); +{% endhighlight %} + +Rich functions can also be defined as an anonymous class: +{% highlight java %} +data.map (new RichMapFunction() { + public Integer map(String value) { return Integer.parseInt(value); } +}); +{% endhighlight %} + +
+
+ + +## Lambda Functions + +As already seen in previous examples all operations accept lambda functions for describing +the operation: +{% highlight scala %} +val data: DataSet[String] = // [...] +data.filter { _.startsWith("http://") } +{% endhighlight %} + +{% highlight scala %} +val data: DataSet[Int] = // [...] +data.reduce { (i1,i2) => i1 + i2 } +// or +data.reduce { _ + _ } +{% endhighlight %} + +## Rich functions + +All transformations that take as argument a lambda function can +instead take as argument a *rich* function. For example, instead of + +{% highlight scala %} +data.map { x => x.toInt } +{% endhighlight %} + +you can write + +{% highlight scala %} +class MyMapFunction extends RichMapFunction[String, Int] { + def map(in: String):Int = { in.toInt } +}; +{% endhighlight %} + +and pass the function to a `map` transformation: + +{% highlight scala %} +data.map(new MyMapFunction()) +{% endhighlight %} + +Rich functions can also be defined as an anonymous class: +{% highlight scala %} +data.map (new RichMapFunction[String, Int] { + def map(in: String):Int = { in.toInt } +}) +{% endhighlight %} +
+ +
+ +Rich functions provide, in addition to the user-defined function (map, +reduce, etc), four methods: `open`, `close`, `getRuntimeContext`, and +`setRuntimeContext`. These are useful for parameterizing the function +(see [Passing Parameters to Functions]({{ site.baseurl }}/dev/batch/index.html#passing-parameters-to-functions)), +creating and finalizing local state, accessing broadcast variables (see +[Broadcast Variables]({{ site.baseurl }}/dev/batch/index.html#broadcast-variables)), and for accessing runtime +information such as accumulators and counters (see +[Accumulators and Counters](#accumulators--counters)), and information +on iterations (see [Iterations]({{ site.baseurl }}/dev/batch/iterations.html)). + +{% top %} + +## Accumulators & Counters + +Accumulators are simple constructs with an **add operation** and a **final accumulated result**, +which is available after the job ended. + +The most straightforward accumulator is a **counter**: You can increment it using the +```Accumulator.add(V value)``` method. At the end of the job Flink will sum up (merge) all partial +results and send the result to the client. Accumulators are useful during debugging or if you +quickly want to find out more about your data. + +Flink currently has the following **built-in accumulators**. Each of them implements the +{% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java "Accumulator" %} +interface. + +- {% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/IntCounter.java "__IntCounter__" %}, + {% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/LongCounter.java "__LongCounter__" %} + and {% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/DoubleCounter.java "__DoubleCounter__" %}: + See below for an example using a counter. +- {% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/Histogram.java "__Histogram__" %}: + A histogram implementation for a discrete number of bins. Internally it is just a map from Integer + to Integer. You can use this to compute distributions of values, e.g. the distribution of + words-per-line for a word count program. + +__How to use accumulators:__ + +First you have to create an accumulator object (here a counter) in the user-defined transformation +function where you want to use it. + +{% highlight java %} +private IntCounter numLines = new IntCounter(); +{% endhighlight %} + +Second you have to register the accumulator object, typically in the ```open()``` method of the +*rich* function. Here you also define the name. + +{% highlight java %} +getRuntimeContext().addAccumulator("num-lines", this.numLines); +{% endhighlight %} + +You can now use the accumulator anywhere in the operator function, including in the ```open()``` and +```close()``` methods. + +{% highlight java %} +this.numLines.add(1); +{% endhighlight %} + +The overall result will be stored in the ```JobExecutionResult``` object which is +returned from the `execute()` method of the execution environment +(currently this only works if the execution waits for the +completion of the job). + +{% highlight java %} +myJobExecutionResult.getAccumulatorResult("num-lines") +{% endhighlight %} + +All accumulators share a single namespace per job. Thus you can use the same accumulator in +different operator functions of your job. Flink will internally merge all accumulators with the same +name. + +A note on accumulators and iterations: Currently the result of accumulators is only available after +the overall job has ended. We plan to also make the result of the previous iteration available in the +next iteration. You can use +{% gh_link /flink-java/src/main/java/org/apache/flink/api/java/operators/IterativeDataSet.java#L98 "Aggregators" %} +to compute per-iteration statistics and base the termination of iterations on such statistics. + +__Custom accumulators:__ + +To implement your own accumulator you simply have to write your implementation of the Accumulator +interface. Feel free to create a pull request if you think your custom accumulator should be shipped +with Flink. + +You have the choice to implement either +{% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java "Accumulator" %} +or {% gh_link /flink-core/src/main/java/org/apache/flink/api/common/accumulators/SimpleAccumulator.java "SimpleAccumulator" %}. + +```Accumulator``` is most flexible: It defines a type ```V``` for the value to add, and a +result type ```R``` for the final result. E.g. for a histogram, ```V``` is a number and ```R``` is + a histogram. ```SimpleAccumulator``` is for the cases where both types are the same, e.g. for counters. + +{% top %} diff --git a/docs/getting-started/docker-playgrounds/flink-operations-playground.md b/docs/getting-started/docker-playgrounds/flink-operations-playground.md index 1e2a569051f2b..6d9f4094c016f 100644 --- a/docs/getting-started/docker-playgrounds/flink-operations-playground.md +++ b/docs/getting-started/docker-playgrounds/flink-operations-playground.md @@ -316,7 +316,7 @@ docker-compose up -d taskmanager When the Master is notified about the new TaskManager, it schedules the tasks of the recovering Job to the newly available TaskSlots. Upon restart, the tasks recover their state from -the last successful [checkpoint]({{ site.baseurl }}/internals/stream_checkpointing.html) that was taken +the last successful [checkpoint]({{ site.baseurl }}/training/fault_tolerance.html) that was taken before the failure and switch to the `RUNNING` state. The Job will quickly process the full backlog of input events (accumulated during the outage) @@ -806,7 +806,7 @@ You might have noticed that the *Click Event Count* application was always start and `--event-time` program arguments. By omitting these in the command of the *client* container in the `docker-compose.yaml`, you can change the behavior of the Job. -* `--checkpointing` enables [checkpoint]({{ site.baseurl }}/internals/stream_checkpointing.html), +* `--checkpointing` enables [checkpoint]({{ site.baseurl }}/training/fault_tolerance.html), which is Flink's fault-tolerance mechanism. If you run without it and go through [failure and recovery](#observing-failure--recovery), you should will see that data is actually lost. diff --git a/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md b/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md index 1e2a569051f2b..6d9f4094c016f 100644 --- a/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md +++ b/docs/getting-started/docker-playgrounds/flink-operations-playground.zh.md @@ -316,7 +316,7 @@ docker-compose up -d taskmanager When the Master is notified about the new TaskManager, it schedules the tasks of the recovering Job to the newly available TaskSlots. Upon restart, the tasks recover their state from -the last successful [checkpoint]({{ site.baseurl }}/internals/stream_checkpointing.html) that was taken +the last successful [checkpoint]({{ site.baseurl }}/training/fault_tolerance.html) that was taken before the failure and switch to the `RUNNING` state. The Job will quickly process the full backlog of input events (accumulated during the outage) @@ -806,7 +806,7 @@ You might have noticed that the *Click Event Count* application was always start and `--event-time` program arguments. By omitting these in the command of the *client* container in the `docker-compose.yaml`, you can change the behavior of the Job. -* `--checkpointing` enables [checkpoint]({{ site.baseurl }}/internals/stream_checkpointing.html), +* `--checkpointing` enables [checkpoint]({{ site.baseurl }}/training/fault_tolerance.html), which is Flink's fault-tolerance mechanism. If you run without it and go through [failure and recovery](#observing-failure--recovery), you should will see that data is actually lost. diff --git a/docs/getting-started/walkthroughs/python_table_api.zh.md b/docs/getting-started/walkthroughs/python_table_api.zh.md index a82ceb3b09f0d..34a0170a267ab 100644 --- a/docs/getting-started/walkthroughs/python_table_api.zh.md +++ b/docs/getting-started/walkthroughs/python_table_api.zh.md @@ -28,7 +28,7 @@ under the License. 在该教程中，我们会从零开始，介绍如何创建一个Flink Python项目及运行Python Table API程序。 -关于Python执行环境的要求，请参考Python Table API[环境安装]({{ site.baseurl }}/dev/dev/table/python/installation.html)。 +关于Python执行环境的要求，请参考Python Table API[环境安装]({{ site.baseurl }}/dev/table/python/installation.html)。 ## 创建一个Python Table API项目 diff --git a/docs/index.md b/docs/index.md index 9f0acae12d89f..09e7f7e93c886 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,9 +36,10 @@ Apache Flink is an open source platform for distributed stream and batch data pr * **Docker Playgrounds**: Set up a sandboxed Flink environment in just a few minutes to explore and play with Flink. * [Run and manage Flink streaming applications](./getting-started/docker-playgrounds/flink-operations-playground.html) -* **Concepts**: Learn about Flink's basic concepts to better understand the documentation. - * [Dataflow Programming Model](concepts/programming-model.html) - * [Distributed Runtime](concepts/runtime.html) +* **Concepts**: Learn about Flink's concepts to better understand the documentation. + * [Stateful Stream Processing](concepts/stateful-stream-processing.html) + * [Timely Stream Processing](concepts/timely-stream-processing.html) + * [Flink Architecture](concepts/flink-architecture.html) * [Glossary](concepts/glossary.html) ## API References diff --git a/docs/index.zh.md b/docs/index.zh.md index 2315024829eab..76e6d45c4a169 100644 --- a/docs/index.zh.md +++ b/docs/index.zh.md @@ -38,8 +38,9 @@ Apache Flink 是一个分布式流批一体化的开源平台。Flink 的核心 * [运行与管理 Flink 流处理应用](./getting-started/docker-playgrounds/flink-operations-playground.html) * **概念**: 学习 Flink 的基本概念能更好地理解文档。 - * [数据流编程模型](concepts/programming-model.html) - * [分布式执行](concepts/runtime.html) + * [有状态流处理](concepts/stateful-stream-processing.html) + * [实时流处理](concepts/timely-stream-processing.html) + * [Flink 架构](concepts/flink-architecture.html) * [术语表](concepts/glossary.html) ## API 参考 diff --git a/docs/internals/task_lifecycle.md b/docs/internals/task_lifecycle.md index 44f847f0cba6d..4d5c485a41055 100644 --- a/docs/internals/task_lifecycle.md +++ b/docs/internals/task_lifecycle.md @@ -92,7 +92,7 @@ operator is opened and before it is closed. The responsibility of this method is to the specified [state backend]({{ site.baseurl }}/ops/state/state_backends.html) from where it will be retrieved when the job resumes execution after a failure. Below we include a brief description of Flink's checkpointing mechanism, and for a more detailed discussion on the principles around checkpointing in Flink please read the corresponding documentation: -[Data Streaming Fault Tolerance]({{ site.baseurl }}/internals/stream_checkpointing.html). +[Data Streaming Fault Tolerance]({{ site.baseurl }}/training/fault_tolerance.html). ## Task Lifecycle diff --git a/docs/internals/task_lifecycle.zh.md b/docs/internals/task_lifecycle.zh.md index 7de935b4ed88f..bc5cccb877889 100644 --- a/docs/internals/task_lifecycle.zh.md +++ b/docs/internals/task_lifecycle.zh.md @@ -92,7 +92,7 @@ operator is opened and before it is closed. The responsibility of this method is to the specified [state backend]({{ site.baseurl }}/ops/state/state_backends.html) from where it will be retrieved when the job resumes execution after a failure. Below we include a brief description of Flink's checkpointing mechanism, and for a more detailed discussion on the principles around checkpointing in Flink please read the corresponding documentation: -[Data Streaming Fault Tolerance]({{ site.baseurl }}/internals/stream_checkpointing.html). +[Data Streaming Fault Tolerance]({{ site.baseurl }}/training/fault_tolerance.html). ## Task Lifecycle diff --git a/docs/monitoring/metrics.zh.md b/docs/monitoring/metrics.zh.md index 04f6fd9ac9fcd..29c6e70f83eee 100644 --- a/docs/monitoring/metrics.zh.md +++ b/docs/monitoring/metrics.zh.md @@ -29,7 +29,7 @@ Flink exposes a metric system that allows gathering and exposing metrics to exte ## Registering metrics -You can access the metric system from any user function that extends [RichFunction]({{ site.baseurl }}/dev/api_concepts.html#rich-functions) by calling `getRuntimeContext().getMetricGroup()`. +You can access the metric system from any user function that extends [RichFunction]({{ site.baseurl }}/zh/dev/user_defined_functions.html#rich-functions) by calling `getRuntimeContext().getMetricGroup()`. This method returns a `MetricGroup` object on which you can create and register new metrics. ### Metric types diff --git a/docs/ops/config.md b/docs/ops/config.md index 950d65bc5d75e..f33d03fda8516 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -158,7 +158,7 @@ In most cases, users should only need to set the values `taskmanager.memory.proc For a detailed explanation of how these options interact, see the documentation on [TaskManager]({{site.baseurl}}/ops/memory/mem_setup_tm.html) and -[JobManager]({{site.baseurl}}/ops/memory/mem_setup_jm.html) memory configurations. +[JobManager]({{site.baseurl}}/ops/memory/mem_setup_master.html) memory configurations. {% include generated/common_memory_section.html %} diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md index 1d3151dbcd87f..80244fd4bb5c9 100644 --- a/docs/ops/config.zh.md +++ b/docs/ops/config.zh.md @@ -158,7 +158,7 @@ In most cases, users should only need to set the values `taskmanager.memory.proc For a detailed explanation of how these options interact, see the documentation on [TaskManager]({{site.baseurl}}/ops/memory/mem_setup_tm.html) and -[JobManager]({{site.baseurl}}/ops/memory/mem_setup_jm.html) memory configurations. +[JobManager]({{site.baseurl}}/ops/memory/mem_setup_master.html) memory configurations. {% include generated/common_memory_section.html %} diff --git a/docs/ops/memory/mem_migration.zh.md b/docs/ops/memory/mem_migration.zh.md index 7c8a525e330c4..72af5469bdf85 100644 --- a/docs/ops/memory/mem_migration.zh.md +++ b/docs/ops/memory/mem_migration.zh.md @@ -119,7 +119,7 @@ Flink 自带的[默认 flink-conf.yaml](#flink-confyaml-中的默认配置) 文 尽管网络内存的配置参数没有发生太多变化，我们仍建议您检查其配置结果。 网络内存的大小可能会受到其他内存部分大小变化的影响，例如总内存变化时，根据占比计算出的网络内存也可能发生变化。 -请参考[内存模型详解](mem_detail.html)。 +请参考[内存模型详解](mem_setup.html)。 容器切除（Cut-Off）内存相关的配置参数（`containerized.heap-cutoff-ratio` 和 `containerized.heap-cutoff-min`）将不再对进程生效。 @@ -153,7 +153,7 @@ Flink 在 Mesos 上还有另一个具有同样语义的配置参数 `mesos.resou 或 [FsStateBackend](../state/state_backends.html#fsstatebackend)），那么它同样需要使用 JVM 堆内存。 Flink 现在总是会预留一部分 JVM 堆内存供框架使用（[`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size)）。 -请参考[框架内存](mem_detail.html#框架内存)。 +请参考[框架内存](mem_setup.html#框架内存)。 ## 托管内存 @@ -201,7 +201,7 @@ Flink 现在总是会预留一部分 JVM 堆内存供框架使用（[`taskmanage * 任务堆外内存（[`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size)） * 框架堆外内存（[`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size)） * JVM Metaspace（[`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size)） -* JVM 开销（请参考[内存模型详解](mem_detail.html)） +* JVM 开销（请参考[内存模型详解](mem_setup_tm.html#detailed-memory-model)） 提示 JobManager 进程仍保留了容器切除内存，相关配置项和此前一样仍对 JobManager 生效。 diff --git a/docs/ops/memory/mem_trouble.zh.md b/docs/ops/memory/mem_trouble.zh.md index 0ecb5ada5deb1..52e08e83b8ed4 100644 --- a/docs/ops/memory/mem_trouble.zh.md +++ b/docs/ops/memory/mem_trouble.zh.md @@ -28,14 +28,14 @@ under the License. ## IllegalConfigurationException 如果遇到从 *TaskExecutorProcessUtils* 抛出的 *IllegalConfigurationException* 异常，这通常说明您的配置参数中存在无效值（例如内存大小为负数、占比大于 1 等）或者配置冲突。 -请根据异常信息，确认[内存模型详解](mem_detail.html)中与出错的内存部分对应章节的内容。 +请根据异常信息，确认[内存模型详解](../config.html#memory-configuration)中与出错的内存部分对应章节的内容。 ## OutOfMemoryError: Java heap space 该异常说明 JVM 的堆空间过小。 可以通过增大[总内存](mem_setup.html#配置总内存)或[任务堆内存](mem_setup.html#任务算子堆内存)的方法来增大 JVM 堆空间。 -提示 也可以增大[框架堆内存](mem_detail.html#框架内存)。这是一个进阶配置，只有在确认是 Flink 框架自身需要更多内存时才应该去调整。 +提示 也可以增大[框架堆内存](mem_setup_tm.html#框架内存)。这是一个进阶配置，只有在确认是 Flink 框架自身需要更多内存时才应该去调整。 ## OutOfMemoryError: Direct buffer memory diff --git a/docs/ops/memory/mem_tuning.zh.md b/docs/ops/memory/mem_tuning.zh.md index 9fb950d8ff138..eac6331af3166 100644 --- a/docs/ops/memory/mem_tuning.zh.md +++ b/docs/ops/memory/mem_tuning.zh.md @@ -30,7 +30,7 @@ under the License. ## 独立部署模式（Standalone Deployment）下的内存配置 [独立部署模式](../deployment/cluster_setup.html)，我们通常更关注 Flink 应用本身使用的内存大小。 -建议配置 [Flink 总内存](mem_setup.html#配置总内存)（[`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)）或者它的[组成部分](mem_detail.html)。 +建议配置 [Flink 总内存](mem_setup.html#配置总内存)（[`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)）或者它的([`jobmanager.memory.flink.size`])(../config.html#jobmanager-memory-flink-size.html)。 此外，如果出现 [Metaspace 不足的问题](mem_trouble.html#outofmemoryerror-metaspace)，可以调整 *JVM Metaspace* 的大小。 这种情况下通常无需配置*进程总内存*，因为不管是 Flink 还是部署环境都不会对 *JVM 开销* 进行限制，它只与机器的物理资源相关。 @@ -41,7 +41,6 @@ under the License. 该配置参数用于指定分配给 Flink *JVM 进程*的总内存，也就是需要申请的容器大小。 提示 如果配置了 *Flink 总内存*，Flink 会自动加上 JVM 相关的内存部分，根据推算出的*进程总内存*大小申请容器。 -请参考[内存模型详解](mem_detail.html)。
注意： 如果 Flink 或者用户代码分配超过容器大小的非托管的堆外（本地）内存，部署环境可能会杀掉超用内存的容器，造成作业执行失败。 diff --git a/docs/ops/python_shell.zh.md b/docs/ops/python_shell.zh.md index e5f2a6c6093c2..2f561c7d70a5e 100644 --- a/docs/ops/python_shell.zh.md +++ b/docs/ops/python_shell.zh.md @@ -27,7 +27,7 @@ Flink附带了一个集成的交互式Python Shell。 本地安装Flink，请看[本地安装](deployment/local.html)页面。 您也可以从源码安装Flink，请看[从源码构建 Flink](../flinkDev/building.html)页面。 -注意 Python Shell会调用“python”命令。关于Python执行环境的要求，请参考Python Table API[环境安装]({{ site.baseurl }}/dev/dev/table/python/installation.html)。 +注意 Python Shell会调用“python”命令。关于Python执行环境的要求，请参考Python Table API[环境安装]({{ site.baseurl }}/dev/table/python/installation.html)。 你可以通过PyPi安装PyFlink，然后使用Python Shell: diff --git a/docs/ops/state/savepoints.md b/docs/ops/state/savepoints.md index c235344eebcfe..d1e07f27e0945 100644 --- a/docs/ops/state/savepoints.md +++ b/docs/ops/state/savepoints.md @@ -27,7 +27,7 @@ under the License. ## What is a Savepoint? How is a Savepoint different from a Checkpoint? -A Savepoint is a consistent image of the execution state of a streaming job, created via Flink's [checkpointing mechanism]({{ site.baseurl }}/internals/stream_checkpointing.html). You can use Savepoints to stop-and-resume, fork, +A Savepoint is a consistent image of the execution state of a streaming job, created via Flink's [checkpointing mechanism]({{ site.baseurl }}/training/fault_tolerance.html). You can use Savepoints to stop-and-resume, fork, or update your Flink jobs. Savepoints consist of two parts: a directory with (typically large) binary files on stable storage (e.g. HDFS, S3, ...) and a (relatively small) meta data file. The files on stable storage represent the net data of the job's execution state image. The meta data file of a Savepoint contains (primarily) pointers to all files on stable storage that are part of the Savepoint, in form of absolute paths. diff --git a/docs/ops/state/savepoints.zh.md b/docs/ops/state/savepoints.zh.md index 6bdb9df74a2a6..b8c52f76f6c93 100644 --- a/docs/ops/state/savepoints.zh.md +++ b/docs/ops/state/savepoints.zh.md @@ -27,7 +27,7 @@ under the License. ## 什么是 Savepoint ？ Savepoint 与 Checkpoint 有什么不同？ -Savepoint 是依据 Flink [checkpointing 机制]({{ site.baseurl }}/zh/internals/stream_checkpointing.html)所创建的流作业执行状态的一致镜像。 你可以使用 Savepoint 进行 Flink 作业的停止与重启、fork 或者更新。 Savepoint 由两部分组成：稳定存储（列入 HDFS，S3，...) 上包含二进制文件的目录（通常很大），和元数据文件（相对较小）。 稳定存储上的文件表示作业执行状态的数据镜像。 Savepoint 的元数据文件以（绝对路径）的形式包含（主要）指向作为 Savepoint 一部分的稳定存储上的所有文件的指针。 +Savepoint 是依据 Flink [checkpointing 机制]({{ site.baseurl }}/zh/training/fault_tolerance.html)所创建的流作业执行状态的一致镜像。 你可以使用 Savepoint 进行 Flink 作业的停止与重启、fork 或者更新。 Savepoint 由两部分组成：稳定存储（列入 HDFS，S3，...) 上包含二进制文件的目录（通常很大），和元数据文件（相对较小）。 稳定存储上的文件表示作业执行状态的数据镜像。 Savepoint 的元数据文件以（绝对路径）的形式包含（主要）指向作为 Savepoint 一部分的稳定存储上的所有文件的指针。
注意: 为了允许程序和 Flink 版本之间的升级，请务必查看以下有关分配算子 ID 的部分 。 From 7c4f1d0c1b552eec1730abe354a4b2c60c487a78 Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Tue, 19 May 2020 15:41:32 +0800 Subject: [PATCH 032/773] [docs-sync] Synchronize the latest documentation changes into Chinese documents --- docs/concepts/flink-architecture.zh.md | 2 +- docs/concepts/glossary.zh.md | 33 ++- docs/dev/batch/hadoop_compatibility.zh.md | 2 +- docs/dev/batch/index.zh.md | 259 ++++++++++++++++-- docs/dev/connectors/cassandra.zh.md | 2 +- docs/dev/connectors/filesystem_sink.zh.md | 2 +- docs/dev/connectors/kafka.zh.md | 2 +- docs/dev/connectors/kinesis.zh.md | 23 +- docs/dev/connectors/nifi.zh.md | 2 +- docs/dev/connectors/pubsub.zh.md | 2 +- docs/dev/connectors/rabbitmq.zh.md | 2 +- docs/dev/connectors/streamfile_sink.zh.md | 201 +++++++++++++- docs/dev/connectors/twitter.zh.md | 2 +- docs/dev/datastream_api.zh.md | 218 ++++++++++++++- docs/dev/java_lambdas.zh.md | 4 +- docs/dev/libs/cep.zh.md | 4 +- docs/dev/parallel.zh.md | 2 +- docs/dev/stream/operators/index.zh.md | 4 +- docs/dev/stream/operators/windows.zh.md | 2 +- docs/dev/stream/state/checkpointing.zh.md | 2 +- docs/dev/stream/state/index.zh.md | 21 +- docs/dev/stream/state/queryable_state.zh.md | 2 +- docs/dev/stream/state/state.zh.md | 130 ++++++--- docs/dev/table/common.zh.md | 4 +- docs/dev/table/connect.zh.md | 42 ++- docs/dev/table/index.zh.md | 2 +- docs/dev/table/sqlClient.zh.md | 4 +- docs/dev/types_serialization.zh.md | 200 ++++++++++++++ docs/dev/user_defined_functions.zh.md | 2 +- docs/getting-started/index.zh.md | 49 +++- .../walkthroughs/python_table_api.zh.md | 27 ++ docs/index.zh.md | 1 - docs/monitoring/metrics.zh.md | 6 +- 33 files changed, 1127 insertions(+), 133 deletions(-) diff --git a/docs/concepts/flink-architecture.zh.md b/docs/concepts/flink-architecture.zh.md index 8414943fd167d..00a09247c07eb 100644 --- a/docs/concepts/flink-architecture.zh.md +++ b/docs/concepts/flink-architecture.zh.md @@ -1,5 +1,5 @@ --- -title: Flink Architecture +title: Flink 架构 nav-id: flink-architecture nav-pos: 4 nav-title: Flink Architecture diff --git a/docs/concepts/glossary.zh.md b/docs/concepts/glossary.zh.md index 8efd2f101a7eb..3f88e82083a77 100644 --- a/docs/concepts/glossary.zh.md +++ b/docs/concepts/glossary.zh.md @@ -25,7 +25,16 @@ under the License. #### Flink Application Cluster -Flink Application Cluster 是一个专用的 [Flink Cluster](#flink-cluster)，它仅用于执行单个 [Flink Job](#flink-job)。[Flink Cluster](#flink-cluster)的生命周期与 [Flink Job](#flink-job)的生命周期绑定在一起。以前，Flink Application Cluster 也称为*job mode*的 Flink Cluster。和 [Flink Session Cluster](#flink-session-cluster) 作对比。 +A Flink Application Cluster is a dedicated [Flink Cluster](#flink-cluster) that +only executes [Flink Jobs](#flink-job) from one [Flink +Application](#flink-application). The lifetime of the [Flink +Cluster](#flink-cluster) is bound to the lifetime of the Flink Application. + +#### Flink Job Cluster + +A Flink Job Cluster is a dedicated [Flink Cluster](#flink-cluster) that only +executes a single [Flink Job](#flink-job). The lifetime of the +[Flink Cluster](#flink-cluster) is bound to the lifetime of the Flink Job. #### Flink Cluster @@ -47,9 +56,22 @@ Function 是由用户实现的，并封装了 Flink 程序的应用程序逻辑 Instance 常用于描述运行时的特定类型(通常是 [Operator](#operator) 或者 [Function](#function))的一个具体实例。由于 Apache Flink 主要是用 Java 编写的，所以，这与 Java 中的 *Instance* 或 *Object* 的定义相对应。在 Apache Flink 的上下文中，*parallel instance* 也常用于强调同一 [Operator](#operator) 或者 [Function](#function) 的多个 instance 以并行的方式运行。 +#### Flink Application + +A Flink application is a Java Application that submits one or multiple [Flink +Jobs](#flink-job) from the `main()` method (or by some other means). Submitting +jobs is usually done by calling `execute()` on an execution environment. + +The jobs of an application can either be submitted to a long running [Flink +Session Cluster](#flink-session-cluster), to a dedicated [Flink Application +Cluster](#flink-application-cluster), or to a [Flink Job +Cluster](#flink-job-cluster). + #### Flink Job -Flink Job 代表运行时的 Flink 程序。Flink Job 可以提交到长时间运行的 [Flink Session Cluster](#flink-session-cluster)，也可以作为独立的 [Flink Application Cluster](#flink-application-cluster) 启动。 +A Flink Job is the runtime representation of a [logical graph](#logical-graph) +(also often called dataflow graph) that is created and submitted by calling +`execute()` in a [Flink Application](#flink-application). #### JobGraph @@ -61,7 +83,12 @@ JobManager 是在 [Flink Master](#flink-master) 运行中的组件之一。JobMa #### Logical Graph -Logical Graph 是一种描述流处理程序的高阶逻辑有向图。节点是[Operator](#operator)，边代表输入/输出关系、数据流和数据集中的之一。 +A logical graph is a directed graph where the nodes are [Operators](#operator) +and the edges define input/output-relationships of the operators and correspond +to data streams or data sets. A logical graph is created by submitting jobs +from a [Flink Application](#flink-application). + +Logical graphs are also often referred to as *dataflow graphs*. #### Managed State diff --git a/docs/dev/batch/hadoop_compatibility.zh.md b/docs/dev/batch/hadoop_compatibility.zh.md index 381ae2fa3ffa1..1f03adbd18e5d 100644 --- a/docs/dev/batch/hadoop_compatibility.zh.md +++ b/docs/dev/batch/hadoop_compatibility.zh.md @@ -28,7 +28,7 @@ reusing code that was implemented for Hadoop MapReduce. You can: -- use Hadoop's `Writable` [data types]({{ site.baseurl }}/dev/api_concepts.html#supported-data-types) in Flink programs. +- use Hadoop's `Writable` [data types]({% link dev/types_serialization.md %}#supported-data-types) in Flink programs. - use any Hadoop `InputFormat` as a [DataSource](index.html#data-sources). - use any Hadoop `OutputFormat` as a [DataSink](index.html#data-sinks). - use a Hadoop `Mapper` as [FlatMapFunction](dataset_transformations.html#flatmap). diff --git a/docs/dev/batch/index.zh.md b/docs/dev/batch/index.zh.md index 55a3be0a51054..66fd0cf3902c6 100644 --- a/docs/dev/batch/index.zh.md +++ b/docs/dev/batch/index.zh.md @@ -32,11 +32,12 @@ example write the data to (distributed) files, or to standard output (for exampl terminal). Flink programs run in a variety of contexts, standalone, or embedded in other programs. The execution can happen in a local JVM, or on clusters of many machines. -Please see [basic concepts]({{ site.baseurl }}/dev/api_concepts.html) for an introduction -to the basic concepts of the Flink API. +Please refer to the [DataStream API overview]({% link dev/datastream_api.md %}) +for an introduction to the basic concepts of the Flink API. That overview is +for the DataStream API but the basic concepts of the two APIs are the same. In order to create your own Flink DataSet program, we encourage you to start with the -[anatomy of a Flink Program]({{ site.baseurl }}/dev/api_concepts.html#anatomy-of-a-flink-program) +[anatomy of a Flink Program]({% link dev/datastream_api.md %}#anatomy-of-a-flink-program) and gradually add your own [transformations](#dataset-transformations). The remaining sections act as references for additional operations and advanced features. @@ -278,7 +279,7 @@ data.distinct(); Joins two data sets by creating all pairs of elements that are equal on their keys. Optionally uses a JoinFunction to turn the pair of elements into a single element, or a FlatJoinFunction to turn the pair of elements into arbitrarily many (including none) - elements. See the keys section to learn how to define join keys. + elements. See the keys section to learn how to define join keys. {% highlight java %} result = input1.join(input2) .where(0) // key of the first input (tuple field 0) @@ -304,7 +305,7 @@ result = input1.join(input2, JoinHint.BROADCAST_HASH_FIRST) OuterJoin - Performs a left, right, or full outer join on two data sets. Outer joins are similar to regular (inner) joins and create all pairs of elements that are equal on their keys. In addition, records of the "outer" side (left, right, or both in case of full) are preserved if no matching key is found in the other side. Matching pairs of elements (or one element and a null value for the other input) are given to a JoinFunction to turn the pair of elements into a single element, or to a FlatJoinFunction to turn the pair of elements into arbitrarily many (including none) elements. See the keys section to learn how to define join keys. + Performs a left, right, or full outer join on two data sets. Outer joins are similar to regular (inner) joins and create all pairs of elements that are equal on their keys. In addition, records of the "outer" side (left, right, or both in case of full) are preserved if no matching key is found in the other side. Matching pairs of elements (or one element and a null value for the other input) are given to a JoinFunction to turn the pair of elements into a single element, or to a FlatJoinFunction to turn the pair of elements into arbitrarily many (including none) elements. See the keys section to learn how to define join keys. {% highlight java %} input1.leftOuterJoin(input2) // rightOuterJoin or fullOuterJoin for right or full outer joins .where(0) // key of the first input (tuple field 0) @@ -326,7 +327,7 @@ input1.leftOuterJoin(input2) // rightOuterJoin or fullOuterJoin for right or ful

The two-dimensional variant of the reduce operation. Groups each input on one or more fields and then joins the groups. The transformation function is called per pair of groups. - See the keys section to learn how to define coGroup keys.

+ See the keys section to learn how to define coGroup keys.

{% highlight java %} data1.coGroup(data2) .where(0) @@ -600,7 +601,7 @@ data.distinct() Joins two data sets by creating all pairs of elements that are equal on their keys. Optionally uses a JoinFunction to turn the pair of elements into a single element, or a FlatJoinFunction to turn the pair of elements into arbitrarily many (including none) - elements. See the keys section to learn how to define join keys. + elements. See the keys section to learn how to define join keys. {% highlight scala %} // In this case tuple fields are used as keys. "0" is the join field on the first tuple // "1" is the join field on the second tuple. @@ -626,7 +627,7 @@ val result = input1.join(input2, JoinHint.BROADCAST_HASH_FIRST) OuterJoin - Performs a left, right, or full outer join on two data sets. Outer joins are similar to regular (inner) joins and create all pairs of elements that are equal on their keys. In addition, records of the "outer" side (left, right, or both in case of full) are preserved if no matching key is found in the other side. Matching pairs of elements (or one element and a `null` value for the other input) are given to a JoinFunction to turn the pair of elements into a single element, or to a FlatJoinFunction to turn the pair of elements into arbitrarily many (including none) elements. See the keys section to learn how to define join keys. + Performs a left, right, or full outer join on two data sets. Outer joins are similar to regular (inner) joins and create all pairs of elements that are equal on their keys. In addition, records of the "outer" side (left, right, or both in case of full) are preserved if no matching key is found in the other side. Matching pairs of elements (or one element and a `null` value for the other input) are given to a JoinFunction to turn the pair of elements into a single element, or to a FlatJoinFunction to turn the pair of elements into arbitrarily many (including none) elements. See the keys section to learn how to define join keys. {% highlight scala %} val joined = left.leftOuterJoin(right).where(0).equalTo(1) { (left, right) => @@ -642,7 +643,7 @@ val joined = left.leftOuterJoin(right).where(0).equalTo(1) {

The two-dimensional variant of the reduce operation. Groups each input on one or more fields and then joins the groups. The transformation function is called per pair of groups. - See the keys section to learn how to define coGroup keys.

+ See the keys section to learn how to define coGroup keys.

{% highlight scala %} data1.coGroup(data2).where(0).equalTo(1) {% endhighlight %} @@ -796,6 +797,230 @@ possible for [Data Sources](#data-sources) and [Data Sinks](#data-sinks). {% top %} +Specifying Keys +--------------- + +Some transformations (join, coGroup, groupBy) require that a key be defined on +a collection of elements. Other transformations (Reduce, GroupReduce, +Aggregate) allow data being grouped on a key before they are +applied. + +A DataSet is grouped as +{% highlight java %} +DataSet<...> input = // [...] +DataSet<...> reduced = input + .groupBy(/*define key here*/) + .reduceGroup(/*do something*/); +{% endhighlight %} + +The data model of Flink is not based on key-value pairs. Therefore, +you do not need to physically pack the data set types into keys and +values. Keys are "virtual": they are defined as functions over the +actual data to guide the grouping operator. + +### Define keys for Tuples +{:.no_toc} + +The simplest case is grouping Tuples on one or more +fields of the Tuple: + +
+
+{% highlight java %} +DataSet> input = // [...] +UnsortedGrouping,Tuple> keyed = input.groupBy(0) +{% endhighlight %} +
+
+{% highlight scala %} +val input: DataSet[(Int, String, Long)] = // [...] +val keyed = input.groupBy(0) +{% endhighlight %} +
+
+ +The tuples are grouped on the first field (the one of +Integer type). + +
+
+{% highlight java %} +DataSet> input = // [...] +UnsortedGrouping,Tuple> keyed = input.groupBy(0,1) +{% endhighlight %} +
+
+{% highlight scala %} +val input: DataSet[(Int, String, Long)] = // [...] +val grouped = input.groupBy(0,1) +{% endhighlight %} +
+
+ +Here, we group the tuples on a composite key consisting of the first and the +second field. + +A note on nested Tuples: If you have a DataSet with a nested tuple, such as: + +{% highlight java %} +DataSet,String,Long>> ds; +{% endhighlight %} + +Specifying `groupBy(0)` will cause the system to use the full `Tuple2` as a key (with the Integer and Float being the key). If you want to "navigate" into the nested `Tuple2`, you have to use field expression keys which are explained below. + +### Define keys using Field Expressions +{:.no_toc} + +You can use String-based field expressions to reference nested fields and define keys for grouping, sorting, joining, or coGrouping. + +Field expressions make it very easy to select fields in (nested) composite types such as [Tuple](#tuples-and-case-classes) and [POJO](#pojos) types. + +
+
+ +In the example below, we have a `WC` POJO with two fields "word" and "count". To group by the field `word`, we just pass its name to the `groupBy()` function. +{% highlight java %} +// some ordinary POJO (Plain old Java Object) +public class WC { + public String word; + public int count; +} +DataSet words = // [...] +DataSet wordCounts = words.groupBy("word") +{% endhighlight %} + +**Field Expression Syntax**: + +- Select POJO fields by their field name. For example `"user"` refers to the "user" field of a POJO type. + +- Select Tuple fields by their field name or 0-offset field index. For example `"f0"` and `"5"` refer to the first and sixth field of a Java Tuple type, respectively. + +- You can select nested fields in POJOs and Tuples. For example `"user.zip"` refers to the "zip" field of a POJO which is stored in the "user" field of a POJO type. Arbitrary nesting and mixing of POJOs and Tuples is supported such as `"f1.user.zip"` or `"user.f3.1.zip"`. + +- You can select the full type using the `"*"` wildcard expressions. This does also work for types which are not Tuple or POJO types. + +**Field Expression Example**: + +{% highlight java %} +public static class WC { + public ComplexNestedClass complex; //nested POJO + private int count; + // getter / setter for private field (count) + public int getCount() { + return count; + } + public void setCount(int c) { + this.count = c; + } +} +public static class ComplexNestedClass { + public Integer someNumber; + public float someFloat; + public Tuple3 word; + public IntWritable hadoopCitizen; +} +{% endhighlight %} + +These are valid field expressions for the example code above: + +- `"count"`: The count field in the `WC` class. + +- `"complex"`: Recursively selects all fields of the field complex of POJO type `ComplexNestedClass`. + +- `"complex.word.f2"`: Selects the last field of the nested `Tuple3`. + +- `"complex.hadoopCitizen"`: Selects the Hadoop `IntWritable` type. + +
+
+ +In the example below, we have a `WC` POJO with two fields "word" and "count". To group by the field `word`, we just pass its name to the `groupBy()` function. +{% highlight scala %} +// some ordinary POJO (Plain old Java Object) +class WC(var word: String, var count: Int) { + def this() { this("", 0L) } +} +val words: DataSet[WC] = // [...] +val wordCounts = words.groupBy("word") + +// or, as a case class, which is less typing +case class WC(word: String, count: Int) +val words: DataSet[WC] = // [...] +val wordCounts = words.groupBy("word") +{% endhighlight %} + +**Field Expression Syntax**: + +- Select POJO fields by their field name. For example `"user"` refers to the "user" field of a POJO type. + +- Select Tuple fields by their 1-offset field name or 0-offset field index. For example `"_1"` and `"5"` refer to the first and sixth field of a Scala Tuple type, respectively. + +- You can select nested fields in POJOs and Tuples. For example `"user.zip"` refers to the "zip" field of a POJO which is stored in the "user" field of a POJO type. Arbitrary nesting and mixing of POJOs and Tuples is supported such as `"_2.user.zip"` or `"user._4.1.zip"`. + +- You can select the full type using the `"_"` wildcard expressions. This does also work for types which are not Tuple or POJO types. + +**Field Expression Example**: + +{% highlight scala %} +class WC(var complex: ComplexNestedClass, var count: Int) { + def this() { this(null, 0) } +} + +class ComplexNestedClass( + var someNumber: Int, + someFloat: Float, + word: (Long, Long, String), + hadoopCitizen: IntWritable) { + def this() { this(0, 0, (0, 0, ""), new IntWritable(0)) } +} +{% endhighlight %} + +These are valid field expressions for the example code above: + +- `"count"`: The count field in the `WC` class. + +- `"complex"`: Recursively selects all fields of the field complex of POJO type `ComplexNestedClass`. + +- `"complex.word._3"`: Selects the last field of the nested `Tuple3`. + +- `"complex.hadoopCitizen"`: Selects the Hadoop `IntWritable` type. + +
+
+ +### Define keys using Key Selector Functions +{:.no_toc} + +An additional way to define keys are "key selector" functions. A key selector function +takes a single element as input and returns the key for the element. The key can be of any type and be derived from deterministic computations. + +The following example shows a key selector function that simply returns the field of an object: + +
+
+{% highlight java %} +// some ordinary POJO +public class WC {public String word; public int count;} +DataSet words = // [...] +UnsortedGrouping keyed = words + .groupBy(new KeySelector() { + public String getKey(WC wc) { return wc.word; } + }); +{% endhighlight %} + +
+
+{% highlight scala %} +// some ordinary case class +case class WC(word: String, count: Int) +val words: DataSet[WC] = // [...] +val keyed = words.groupBy( _.word ) +{% endhighlight %} +
+
+ +{% top %} + Data Sources ------------ @@ -1199,7 +1424,7 @@ myResult.output( #### Locally Sorted Output -The output of a data sink can be locally sorted on specified fields in specified orders using [tuple field positions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-for-tuples) or [field expressions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-using-field-expressions). This works for every output format. +The output of a data sink can be locally sorted on specified fields in specified orders using [tuple field positions](#define-keys-for-tuples) or [field expressions](#define-keys-using-field-expressions). This works for every output format. The following examples show how to use this feature: @@ -1282,7 +1507,7 @@ values map { tuple => tuple._1 + " - " + tuple._2 } #### Locally Sorted Output -The output of a data sink can be locally sorted on specified fields in specified orders using [tuple field positions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-for-tuples) or [field expressions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-using-field-expressions). This works for every output format. +The output of a data sink can be locally sorted on specified fields in specified orders using [tuple field positions](#define-keys-for-tuples) or [field expressions](#define-keys-using-field-expressions). This works for every output format. The following examples show how to use this feature: @@ -1771,7 +1996,7 @@ This information is used by the optimizer to infer whether a data property such partitioning is preserved by a function. For functions that operate on groups of input elements such as `GroupReduce`, `GroupCombine`, `CoGroup`, and `MapPartition`, all fields that are defined as forwarded fields must always be jointly forwarded from the same input element. The forwarded fields of each element that is emitted by a group-wise function may originate from a different element of the function's input group. -Field forward information is specified using [field expressions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-using-field-expressions). +Field forward information is specified using [field expressions](#define-keys-using-field-expressions). Fields that are forwarded to the same position in the output can be specified by their position. The specified position must be valid for the input and output data type and have the same type. For example the String `"f2"` declares that the third field of a Java input tuple is always equal to the third field in the output tuple. @@ -1840,7 +2065,7 @@ Non-forwarded field information for group-wise operators such as `GroupReduce`, **IMPORTANT**: The specification of non-forwarded fields information is optional. However if used, **ALL!** non-forwarded fields must be specified, because all other fields are considered to be forwarded in place. It is safe to declare a forwarded field as non-forwarded. -Non-forwarded fields are specified as a list of [field expressions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-using-field-expressions). The list can be either given as a single String with field expressions separated by semicolons or as multiple Strings. +Non-forwarded fields are specified as a list of [field expressions](#define-keys-using-field-expressions). The list can be either given as a single String with field expressions separated by semicolons or as multiple Strings. For example both `"f1; f3"` and `"f1", "f3"` declare that the second and fourth field of a Java tuple are not preserved in place and all other fields are preserved in place. Non-forwarded field information can only be specified for functions which have identical input and output types. @@ -1891,7 +2116,7 @@ Fields which are only unmodified forwarded to the output without evaluating thei **IMPORTANT**: The specification of read fields information is optional. However if used, **ALL!** read fields must be specified. It is safe to declare a non-read field as read. -Read fields are specified as a list of [field expressions]({{ site.baseurl }}/dev/api_concepts.html#define-keys-using-field-expressions). The list can be either given as a single String with field expressions separated by semicolons or as multiple Strings. +Read fields are specified as a list of [field expressions](#define-keys-using-field-expressions). The list can be either given as a single String with field expressions separated by semicolons or as multiple Strings. For example both `"f1; f3"` and `"f1", "f3"` declare that the second and fourth field of a Java tuple are read and evaluated by the function. Read field information is specified as function class annotations using the following annotations: @@ -2045,7 +2270,7 @@ DataSet result = input.map(new MyMapper()); env.execute(); {% endhighlight %} -Access the cached file or directory in a user function (here a `MapFunction`). The function must extend a [RichFunction]({{ site.baseurl }}/dev/api_concepts.html#rich-functions) class because it needs access to the `RuntimeContext`. +Access the cached file or directory in a user function (here a `MapFunction`). The function must extend a [RichFunction]({% link dev/user_defined_functions.md %}#rich-functions) class because it needs access to the `RuntimeContext`. {% highlight java %} @@ -2091,7 +2316,7 @@ val result: DataSet[Integer] = input.map(new MyMapper()) env.execute() {% endhighlight %} -Access the cached file in a user function (here a `MapFunction`). The function must extend a [RichFunction]({{ site.baseurl }}/dev/api_concepts.html#rich-functions) class because it needs access to the `RuntimeContext`. +Access the cached file in a user function (here a `MapFunction`). The function must extend a [RichFunction]({% link dev/user_defined_functions.md %}#rich-functions) class because it needs access to the `RuntimeContext`. {% highlight scala %} @@ -2164,7 +2389,7 @@ class MyFilter(limit: Int) extends FilterFunction[Int] { #### Via `withParameters(Configuration)` -This method takes a Configuration object as an argument, which will be passed to the [rich function]({{ site.baseurl }}/dev/api_concepts.html#rich-functions)'s `open()` +This method takes a Configuration object as an argument, which will be passed to the [rich function]({% link dev/user_defined_functions.md %}#rich-functions)'s `open()` method. The Configuration object is a Map from String keys to different value types.
diff --git a/docs/dev/connectors/cassandra.zh.md b/docs/dev/connectors/cassandra.zh.md index 9a51387ea2b7a..002d388bcef77 100644 --- a/docs/dev/connectors/cassandra.zh.md +++ b/docs/dev/connectors/cassandra.zh.md @@ -111,7 +111,7 @@ More details on [checkpoints docs]({{ site.baseurl }}/dev/stream/state/checkpoin ## Examples -The Cassandra sinks currently support both Tuple and POJO data types, and Flink automatically detects which type of input is used. For general use case of those streaming data type, please refer to [Supported Data Types]({{ site.baseurl }}/dev/api_concepts.html). We show two implementations based on [SocketWindowWordCount](https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/socket/SocketWindowWordCount.java), for Pojo and Tuple data types respectively. +The Cassandra sinks currently support both Tuple and POJO data types, and Flink automatically detects which type of input is used. For general use case of those streaming data type, please refer to [Supported Data Types]({% link dev/types_serialization.md %}#supported-data-types). We show two implementations based on [SocketWindowWordCount](https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/socket/SocketWindowWordCount.java), for Pojo and Tuple data types respectively. In all these examples, we assumed the associated Keyspace `example` and Table `wordcount` have been created. diff --git a/docs/dev/connectors/filesystem_sink.zh.md b/docs/dev/connectors/filesystem_sink.zh.md index 1e570e684123c..adb9f8be895a1 100644 --- a/docs/dev/connectors/filesystem_sink.zh.md +++ b/docs/dev/connectors/filesystem_sink.zh.md @@ -39,7 +39,7 @@ under the License. {% endhighlight %} -注意连接器目前还不是二进制发行版的一部分，添加依赖、打包配置以及集群运行信息请参考 [这里]({{site.baseurl}}/zh/dev/projectsetup/dependencies.html)。 +注意连接器目前还不是二进制发行版的一部分，添加依赖、打包配置以及集群运行信息请参考 [这里]({{site.baseurl}}/zh/getting-started/project-setup/dependencies.html)。 #### 分桶文件 Sink diff --git a/docs/dev/connectors/kafka.zh.md b/docs/dev/connectors/kafka.zh.md index b2cdc54ec56c0..bf2ebbed7f873 100644 --- a/docs/dev/connectors/kafka.zh.md +++ b/docs/dev/connectors/kafka.zh.md @@ -84,7 +84,7 @@ Flink 提供了专门的 Kafka 连接器，向 Kafka topic 中读取或者写入 {% endhighlight %} 请注意：目前流连接器还不是二进制分发的一部分。 -[在此处]({{ site.baseurl }}/zh/dev/projectsetup/dependencies.html)可以了解到如何链接它们以实现在集群中执行。 +[在此处]({{ site.baseurl }}/zh/getting-started/project-setup/dependencies.html)可以了解到如何链接它们以实现在集群中执行。 ## 安装 Apache Kafka diff --git a/docs/dev/connectors/kinesis.zh.md b/docs/dev/connectors/kinesis.zh.md index 4e9009da5ee22..9601af298d158 100644 --- a/docs/dev/connectors/kinesis.zh.md +++ b/docs/dev/connectors/kinesis.zh.md @@ -45,7 +45,25 @@ Due to the licensing issue, the `flink-connector-kinesis{{ site.scala_version_su ## Using the Amazon Kinesis Streams Service Follow the instructions from the [Amazon Kinesis Streams Developer Guide](https://docs.aws.amazon.com/streams/latest/dev/learning-kinesis-module-one-create-stream.html) -to setup Kinesis streams. Make sure to create the appropriate IAM policy and user to read / write to the Kinesis streams. +to setup Kinesis streams. + +## Configuring Access to Kinesis with IAM +Make sure to create the appropriate IAM policy to allow reading / writing to / from the Kinesis streams. See examples [here](https://docs.aws.amazon.com/streams/latest/dev/controlling-access.html). + +Depending on your deployment you would choose a different Credentials Provider to allow access to Kinesis. +By default, the `AUTO` Credentials Provider is used. +If the access key ID and secret key are set in the configuration, the `BASIC` provider is used. + +A specific Credentials Provider can **optionally** be set by using the `AWSConfigConstants.AWS_CREDENTIALS_PROVIDER` setting. + +Supported Credential Providers are: +* `AUTO` - Using the default AWS Credentials Provider chain that searches for credentials in the following order: `ENV_VARS`, `SYS_PROPS`, `WEB_IDENTITY_TOKEN`, `PROFILE` and EC2/ECS credentials provider. +* `BASIC` - Using access key ID and secret key supplied as configuration. +* `ENV_VAR` - Using `AWS_ACCESS_KEY_ID` & `AWS_SECRET_ACCESS_KEY` environment variables. +* `SYS_PROP` - Using Java system properties aws.accessKeyId and aws.secretKey. +* `PROFILE` - Use AWS credentials profile file to create the AWS credentials. +* `ASSUME_ROLE` - Create AWS credentials by assuming a role. The credentials for assuming the role must be supplied. +* `WEB_IDENTITY_TOKEN` - Create AWS credentials by assuming a role using Web Identity Token. ## Kinesis Consumer @@ -91,8 +109,7 @@ The above is a simple example of using the consumer. Configuration for the consu instance, the configuration keys for which can be found in `AWSConfigConstants` (AWS-specific parameters) and `ConsumerConfigConstants` (Kinesis consumer parameters). The example demonstrates consuming a single Kinesis stream in the AWS region "us-east-1". The AWS credentials are supplied using the basic method in which -the AWS access key ID and secret access key are directly supplied in the configuration (other options are setting -`AWSConfigConstants.AWS_CREDENTIALS_PROVIDER` to `ENV_VAR`, `SYS_PROP`, `PROFILE`, `ASSUME_ROLE`, and `AUTO`). Also, data is being consumed +the AWS access key ID and secret access key are directly supplied in the configuration. Also, data is being consumed from the newest position in the Kinesis stream (the other option will be setting `ConsumerConfigConstants.STREAM_INITIAL_POSITION` to `TRIM_HORIZON`, which lets the consumer start reading the Kinesis stream from the earliest record possible). diff --git a/docs/dev/connectors/nifi.zh.md b/docs/dev/connectors/nifi.zh.md index 114092f9a3306..36ac3f3f5ec4a 100644 --- a/docs/dev/connectors/nifi.zh.md +++ b/docs/dev/connectors/nifi.zh.md @@ -34,7 +34,7 @@ under the License. {% endhighlight %} -注意这些连接器目前还没有包含在二进制发行版中。添加依赖、打包配置以及集群运行的相关信息请参考 [这里]({{site.baseurl}}/zh/dev/projectsetup/dependencies.html)。 +注意这些连接器目前还没有包含在二进制发行版中。添加依赖、打包配置以及集群运行的相关信息请参考 [这里]({{site.baseurl}}/zh/getting-started/project-setup/dependencies.html)。 #### 安装 Apache NiFi diff --git a/docs/dev/connectors/pubsub.zh.md b/docs/dev/connectors/pubsub.zh.md index eaf5f582c8467..93cfa96920a2e 100644 --- a/docs/dev/connectors/pubsub.zh.md +++ b/docs/dev/connectors/pubsub.zh.md @@ -37,7 +37,7 @@ under the License. 注意：此连接器最近才加到 Flink 里，还未接受广泛测试。

-注意连接器目前还不是二进制发行版的一部分，添加依赖、打包配置以及集群运行信息请参考[这里]({{ site.baseurl }}/zh/dev/projectsetup/dependencies.html) +注意连接器目前还不是二进制发行版的一部分，添加依赖、打包配置以及集群运行信息请参考[这里]({{ site.baseurl }}/zh/getting-started/project-setup/dependencies.html) ## Consuming or Producing PubSubMessages diff --git a/docs/dev/connectors/rabbitmq.zh.md b/docs/dev/connectors/rabbitmq.zh.md index e213d3f99b19a..26b16ba476359 100644 --- a/docs/dev/connectors/rabbitmq.zh.md +++ b/docs/dev/connectors/rabbitmq.zh.md @@ -43,7 +43,7 @@ Flink 自身既没有复用 "RabbitMQ AMQP Java Client" 的代码，也没有将 {% endhighlight %} -注意连接器现在没有包含在二进制发行版中。集群执行的相关信息请参考 [这里]({{site.baseurl}}/zh/dev/projectsetup/dependencies.html). +注意连接器现在没有包含在二进制发行版中。集群执行的相关信息请参考 [这里]({{site.baseurl}}/zh/getting-started/project-setup/dependencies.html). ### 安装 RabbitMQ 安装 RabbitMQ 请参考 [RabbitMQ 下载页面](http://www.rabbitmq.com/download.html)。安装完成之后，服务会自动拉起，应用程序就可以尝试连接到 RabbitMQ 了。 diff --git a/docs/dev/connectors/streamfile_sink.zh.md b/docs/dev/connectors/streamfile_sink.zh.md index bd74bef529053..9f027a7211384 100644 --- a/docs/dev/connectors/streamfile_sink.zh.md +++ b/docs/dev/connectors/streamfile_sink.zh.md @@ -122,11 +122,12 @@ input.addSink(sink) 批量编码 Sink 的创建与行编码 Sink 相似，不过在这里我们不是指定编码器 `Encoder` 而是指定 [BulkWriter.Factory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/api/common/serialization/BulkWriter.Factory.html) 。 `BulkWriter` 定义了如何添加、刷新元素，以及如何批量编码。 -Flink 有三个内置的 BulkWriter Factory ： +Flink 有四个内置的 BulkWriter Factory ： - [ParquetWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/parquet/ParquetWriterFactory.html) - [SequenceFileWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/sequencefile/SequenceFileWriterFactory.html) - [CompressWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/compress/CompressWriterFactory.html) + - [OrcBulkWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/orc/writer/OrcBulkWriterFactory.html)
重要: 批量编码模式仅支持 OnCheckpointRollingPolicy 策略, 在每次 checkpoint 的时候切割文件。 @@ -188,6 +189,204 @@ input.addSink(sink)
+#### ORC Format + +To enable the data to be bulk encoded in ORC format, Flink offers [OrcBulkWriterFactory]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/formats/orc/writers/OrcBulkWriterFactory.html) +which takes a concrete implementation of [Vectorizer]({{ site.javadocs_baseurl }}/api/java/org/apache/flink/orc/vector/Vectorizer.html). + +Like any other columnar format that encodes data in bulk fashion, Flink's `OrcBulkWriter` writes the input elements in batches. It uses +ORC's `VectorizedRowBatch` to achieve this. + +Since the input element has to be transformed to a `VectorizedRowBatch`, users have to extend the abstract `Vectorizer` +class and override the `vectorize(T element, VectorizedRowBatch batch)` method. As you can see, the method provides an +instance of `VectorizedRowBatch` to be used directly by the users so users just have to write the logic to transform the +input `element` to `ColumnVectors` and set them in the provided `VectorizedRowBatch` instance. + +For example, if the input element is of type `Person` which looks like: + +
+
+{% highlight java %} + +class Person { + private final String name; + private final int age; + ... +} + +{% endhighlight %} +
+ +Then a child implementation to convert the element of type `Person` and set them in the `VectorizedRowBatch` can be like: + +
+
+{% highlight java %} +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; + +public class PersonVectorizer extends Vectorizer implements Serializable { + public PersonVectorizer(String schema) { + super(schema); + } + @Override + public void vectorize(Person element, VectorizedRowBatch batch) throws IOException { + BytesColumnVector nameColVector = (BytesColumnVector) batch.cols[0]; + LongColumnVector ageColVector = (LongColumnVector) batch.cols[1]; + int row = batch.size++; + nameColVector.setVal(row, element.getName().getBytes(StandardCharsets.UTF_8)); + ageColVector.vector[row] = element.getAge(); + } +} + +{% endhighlight %} +
+
+{% highlight scala %} +import java.nio.charset.StandardCharsets +import org.apache.hadoop.hive.ql.exec.vector.{BytesColumnVector, LongColumnVector} + +class PersonVectorizer(schema: String) extends Vectorizer[Person](schema) { + + override def vectorize(element: Person, batch: VectorizedRowBatch): Unit = { + val nameColVector = batch.cols(0).asInstanceOf[BytesColumnVector] + val ageColVector = batch.cols(1).asInstanceOf[LongColumnVector] + nameColVector.setVal(batch.size + 1, element.getName.getBytes(StandardCharsets.UTF_8)) + ageColVector.vector(batch.size + 1) = element.getAge + } + +} + +{% endhighlight %} +
+
+ +To use the ORC bulk encoder in an application, users need to add the following dependency: + +{% highlight xml %} + + org.apache.flink + flink-orc{{ site.scala_version_suffix }} + {{ site.version }} + +{% endhighlight %} + +And then a `StreamingFileSink` that writes data in ORC format can be created like this: + +
+
+{% highlight java %} +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; +import org.apache.flink.orc.writer.OrcBulkWriterFactory; + +String schema = "struct<_col0:string,_col1:int>"; +DataStream stream = ...; + +final OrcBulkWriterFactory writerFactory = new OrcBulkWriterFactory<>(new PersonVectorizer(schema)); + +final StreamingFileSink sink = StreamingFileSink + .forBulkFormat(outputBasePath, writerFactory) + .build(); + +input.addSink(sink); + +{% endhighlight %} +
+
+{% highlight scala %} +import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink +import org.apache.flink.orc.writer.OrcBulkWriterFactory + +val schema: String = "struct<_col0:string,_col1:int>" +val input: DataStream[Person] = ... +val writerFactory = new OrcBulkWriterFactory(new PersonVectorizer(schema)); + +val sink: StreamingFileSink[Person] = StreamingFileSink + .forBulkFormat(outputBasePath, writerFactory) + .build() + +input.addSink(sink) + +{% endhighlight %} +
+
+ +OrcBulkWriterFactory can also take Hadoop `Configuration` and `Properties` so that a custom Hadoop configuration and ORC +writer properties can be provided. + +
+
+{% highlight java %} +String schema = ...; +Configuration conf = ...; +Properties writerProperties = new Properties(); + +writerProps.setProperty("orc.compress", "LZ4"); +// Other ORC supported properties can also be set similarly. + +final OrcBulkWriterFactory writerFactory = new OrcBulkWriterFactory<>( + new PersonVectorizer(schema), writerProperties, conf); + +{% endhighlight %} +
+
+{% highlight scala %} +val schema: String = ... +val conf: Configuration = ... +val writerProperties: Properties = new Properties() + +writerProps.setProperty("orc.compress", "LZ4") +// Other ORC supported properties can also be set similarly. + +val writerFactory = new OrcBulkWriterFactory( + new PersonVectorizer(schema), writerProperties, conf) +{% endhighlight %} +
+
+ +The complete list of ORC writer properties can be found [here](https://orc.apache.org/docs/hive-config.html). + +Users who want to add user metadata to the ORC files can do so by calling `addUserMetadata(...)` inside the overriding +`vectorize(...)` method. + +
+
+{% highlight java %} + +public class PersonVectorizer extends Vectorizer implements Serializable { + @Override + public void vectorize(Person element, VectorizedRowBatch batch) throws IOException { + ... + String metadataKey = ...; + ByteBuffer metadataValue = ...; + this.addUserMetadata(metadataKey, metadataValue); + } +} + +{% endhighlight %} +
+
+{% highlight scala %} + +class PersonVectorizer(schema: String) extends Vectorizer[Person](schema) { + + override def vectorize(element: Person, batch: VectorizedRowBatch): Unit = { + ... + val metadataKey: String = ... + val metadataValue: ByteBuffer = ... + addUserMetadata(metadataKey, metadataValue) + } + +} + +{% endhighlight %} +
+
+ #### Hadoop SequenceFile 格式 在应用中使用 SequenceFile 批量编码器，你需要添加以下依赖： diff --git a/docs/dev/connectors/twitter.zh.md b/docs/dev/connectors/twitter.zh.md index f4d110d44c024..afd3ba84d12cb 100644 --- a/docs/dev/connectors/twitter.zh.md +++ b/docs/dev/connectors/twitter.zh.md @@ -35,7 +35,7 @@ Flink Streaming 通过一个内置的 `TwitterSource` 类来创建到 tweets 流 {% endhighlight %} -注意：当前的二进制发行版还没有这些连接器。集群执行请参考[这里]({{site.baseurl}}/zh/dev/projectsetup/dependencies.html). +注意：当前的二进制发行版还没有这些连接器。集群执行请参考[这里]({{site.baseurl}}/zh/getting-started/project-setup/dependencies.html). #### 认证 使用 Twitter 流，用户需要先注册自己的程序，获取认证相关的必要信息。过程如下： diff --git a/docs/dev/datastream_api.zh.md b/docs/dev/datastream_api.zh.md index c95a9101dcd01..8300827cfa7dd 100644 --- a/docs/dev/datastream_api.zh.md +++ b/docs/dev/datastream_api.zh.md @@ -32,19 +32,221 @@ example write the data to files, or to standard output (for example the command terminal). Flink programs run in a variety of contexts, standalone, or embedded in other programs. The execution can happen in a local JVM, or on clusters of many machines. -Please see [basic concepts]({{ site.baseurl }}/dev/api_concepts.html) for an introduction -to the basic concepts of the Flink API. - -In order to create your own Flink DataStream program, we encourage you to start with -[anatomy of a Flink Program]({{ site.baseurl }}/dev/api_concepts.html#anatomy-of-a-flink-program) -and gradually add your own -[stream transformations]({{ site.baseurl }}/dev/stream/operators/index.html). The remaining sections act as references for additional -operations and advanced features. +In order to create your own Flink DataStream program, we encourage you to start +with [anatomy of a Flink Program](#anatomy-of-a-flink-program) and gradually +add your own [stream transformations]({{ site.baseurl +}}/dev/stream/operators/index.html). The remaining sections act as references +for additional operations and advanced features. * This will be replaced by the TOC {:toc} +What is a DataStream? +---------------------- + +The DataStream API gets its name from the special `DataStream` class that is +used to represent a collection of data in a Flink program. You can think of +them as immutable collections of data that can contain duplicates. This data +can either be finite or unbounded, the API that you use to work on them is the +same. + +A `DataStream` is similar to a regular Java `Collection` in terms of usage but +is quite different in some key ways. They are immutable, meaning that once they +are created you cannot add or remove elements. You can also not simply inspect +the elements inside but only work on them using the `DataStream` API +operations, which are also called transformations. + +You can create an initial `DataStream` by adding a source in a Flink program. +Then you can derive new streams from this and combine them by using API methods +such as `map`, `filter`, and so on. + +Anatomy of a Flink Program +-------------------------- + +Flink programs look like regular programs that transform `DataStreams`. Each +program consists of the same basic parts: + +1. Obtain an `execution environment`, +2. Load/create the initial data, +3. Specify transformations on this data, +4. Specify where to put the results of your computations, +5. Trigger the program execution + + +
+
+ + +We will now give an overview of each of those steps, please refer to the +respective sections for more details. Note that all core classes of the Java +DataStream API can be found in {% gh_link +/flink-streaming-java/src/main/java/org/apache/flink/streaming/api +"org.apache.flink.streaming.api" %}. + +The `StreamExecutionEnvironment` is the basis for all Flink programs. You can +obtain one using these static methods on `StreamExecutionEnvironment`: + +{% highlight java %} +getExecutionEnvironment() + +createLocalEnvironment() + +createRemoteEnvironment(String host, int port, String... jarFiles) +{% endhighlight %} + +Typically, you only need to use `getExecutionEnvironment()`, since this will do +the right thing depending on the context: if you are executing your program +inside an IDE or as a regular Java program it will create a local environment +that will execute your program on your local machine. If you created a JAR file +from your program, and invoke it through the [command line]({{ site.baseurl +}}/ops/cli.html), the Flink cluster manager will execute your main method and +`getExecutionEnvironment()` will return an execution environment for executing +your program on a cluster. + +For specifying data sources the execution environment has several methods to +read from files using various methods: you can just read them line by line, as +CSV files, or using any of the other provided sources. To just read a text file +as a sequence of lines, you can use: + +{% highlight java %} +final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + +DataStream text = env.readTextFile("file:///path/to/file"); +{% endhighlight %} + +This will give you a DataStream on which you can then apply transformations to create new +derived DataStreams. + +You apply transformations by calling methods on DataStream with a +transformation functions. For example, a map transformation looks like this: + +{% highlight java %} +DataStream input = ...; + +DataStream parsed = input.map(new MapFunction() { + @Override + public Integer map(String value) { + return Integer.parseInt(value); + } +}); +{% endhighlight %} + +This will create a new DataStream by converting every String in the original +collection to an Integer. + +Once you have a DataStream containing your final results, you can write it to +an outside system by creating a sink. These are just some example methods for +creating a sink: + +{% highlight java %} +writeAsText(String path) + +print() +{% endhighlight %} + +
+
+ +We will now give an overview of each of those steps, please refer to the +respective sections for more details. Note that all core classes of the Scala +DataStream API can be found in {% gh_link +/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala +"org.apache.flink.streaming.api.scala" %}. + +The `StreamExecutionEnvironment` is the basis for all Flink programs. You can +obtain one using these static methods on `StreamExecutionEnvironment`: + +{% highlight scala %} +getExecutionEnvironment() + +createLocalEnvironment() + +createRemoteEnvironment(host: String, port: Int, jarFiles: String*) +{% endhighlight %} + +Typically, you only need to use `getExecutionEnvironment()`, since this will do +the right thing depending on the context: if you are executing your program +inside an IDE or as a regular Java program it will create a local environment +that will execute your program on your local machine. If you created a JAR file +from your program, and invoke it through the [command line]({{ site.baseurl +}}/ops/cli.html), the Flink cluster manager will execute your main method and +`getExecutionEnvironment()` will return an execution environment for executing +your program on a cluster. + +For specifying data sources the execution environment has several methods to +read from files using various methods: you can just read them line by line, as +CSV files, or using any of the other provided sources. To just read a text file +as a sequence of lines, you can use: + +{% highlight scala %} +val env = StreamExecutionEnvironment.getExecutionEnvironment() + +val text: DataStream[String] = env.readTextFile("file:///path/to/file") +{% endhighlight %} + +This will give you a DataStream on which you can then apply transformations to +create new derived DataStreams. + +You apply transformations by calling methods on DataStream with a +transformation functions. For example, a map transformation looks like this: + +{% highlight scala %} +val input: DataSet[String] = ... + +val mapped = input.map { x => x.toInt } +{% endhighlight %} + +This will create a new DataStream by converting every String in the original +collection to an Integer. + +Once you have a DataStream containing your final results, you can write it to +an outside system by creating a sink. These are just some example methods for +creating a sink: + +{% highlight scala %} +writeAsText(path: String) + +print() +{% endhighlight %} + +
+
+ +Once you specified the complete program you need to **trigger the program +execution** by calling `execute()` on the `StreamExecutionEnvironment`. +Depending on the type of the `ExecutionEnvironment` the execution will be +triggered on your local machine or submit your program for execution on a +cluster. + +The `execute()` method will wait for the job to finish and then return a +`JobExecutionResult`, this contains execution times and accumulator results. + +If you don't want to wait for the job to finish, you can trigger asynchronous +job execution by calling `executeAysnc()` on the `StreamExecutionEnvironment`. +It will return a `JobClient` with which you can communicate with the job you +just submitted. For instance, here is how to implement the semantics of +`execute()` by using `executeAsync()`. + +{% highlight java %} +final JobClient jobClient = env.executeAsync(); + +final JobExecutionResult jobExecutionResult = jobClient.getJobExecutionResult(userClassloader).get(); +{% endhighlight %} + +That last part about program execution is crucial to understanding when and how +Flink operations are executed. All Flink programs are executed lazily: When the +program's main method is executed, the data loading and transformations do not +happen directly. Rather, each operation is created and added to a dataflow +graph. The operations are actually executed when the execution is explicitly +triggered by an `execute()` call on the execution environment. Whether the +program is executed locally or on a cluster depends on the type of execution +environment + +The lazy evaluation lets you construct sophisticated programs that Flink +executes as one holistically planned unit. + +{% top %} Example Program --------------- diff --git a/docs/dev/java_lambdas.zh.md b/docs/dev/java_lambdas.zh.md index 410a66afc9cb6..8fbc6bcb708db 100644 --- a/docs/dev/java_lambdas.zh.md +++ b/docs/dev/java_lambdas.zh.md @@ -26,7 +26,9 @@ Java 8 引入了几种新的语言特性，旨在实现更快、更清晰的编 注意 Flink 支持对 Java API 的所有算子使用 Lambda 表达式，但是，当 Lambda 表达式使用 Java 泛型时，你需要 *显式* 声明类型信息。 -本文档介绍了如何使用 Lambda 表达式并描述了其在当前应用中的限制。有关 Flink API 的通用介绍， 请参阅[编程指南]({{ site.baseurl }}/zh/dev/api_concepts.html)。 +This document shows how to use lambda expressions and describes current +limitations. For a general introduction to the Flink API, please refer to the +[DataSteam API overview]({{ site.baseurl }}{% link dev/datastream_api.zh.md %}) ### 示例和限制 diff --git a/docs/dev/libs/cep.zh.md b/docs/dev/libs/cep.zh.md index 3805282d547e7..4a2a06c9e8a19 100644 --- a/docs/dev/libs/cep.zh.md +++ b/docs/dev/libs/cep.zh.md @@ -35,7 +35,7 @@ FlinkCEP是在Flink上层实现的复杂事件处理库。 ## 开始 -如果你想现在开始尝试，[创建一个Flink程序]({{ site.baseurl }}/zh/dev/projectsetup/dependencies.html)， +如果你想现在开始尝试，[创建一个Flink程序]({{ site.baseurl }}/zh/getting-started/project-setup/dependencies.html)， 添加FlinkCEP的依赖到项目的`pom.xml`文件中。
@@ -60,7 +60,7 @@ FlinkCEP是在Flink上层实现的复杂事件处理库。
-{% info 提示 %} FlinkCEP不是二进制发布包的一部分。在集群上执行如何链接它可以看[这里]({{site.baseurl}}/zh/dev/projectsetup/dependencies.html)。 +{% info 提示 %} FlinkCEP不是二进制发布包的一部分。在集群上执行如何链接它可以看[这里]({{site.baseurl}}/zh/getting-started/project-setup/dependencies.html)。 现在可以开始使用Pattern API写你的第一个CEP程序了。 diff --git a/docs/dev/parallel.zh.md b/docs/dev/parallel.zh.md index c1b603876de65..041136dce76ea 100644 --- a/docs/dev/parallel.zh.md +++ b/docs/dev/parallel.zh.md @@ -73,7 +73,7 @@ env.execute("Word Count Example") ### 执行环境层次 -如[此节]({{ site.baseurl }}/zh/dev/api_concepts.html#anatomy-of-a-flink-program)所描述，Flink 程序运行在执行环境的上下文中。执行环境为所有执行的算子、数据源、数据接收器 (data sink) 定义了一个默认的并行度。可以显式配置算子层次的并行度去覆盖执行环境的并行度。 +如[此节]({{ site.baseurl }}{% link dev/datastream_api.zh.md %}#anatomy-of-a-flink-program)所描述，Flink 程序运行在执行环境的上下文中。执行环境为所有执行的算子、数据源、数据接收器 (data sink) 定义了一个默认的并行度。可以显式配置算子层次的并行度去覆盖执行环境的并行度。 可以通过调用 `setParallelism()` 方法指定执行环境的默认并行度。如果想以并行度`3`来执行所有的算子、数据源和数据接收器。可以在执行环境上设置默认并行度，如下所示： diff --git a/docs/dev/stream/operators/index.zh.md b/docs/dev/stream/operators/index.zh.md index 2f54335234228..d226b01bce864 100644 --- a/docs/dev/stream/operators/index.zh.md +++ b/docs/dev/stream/operators/index.zh.md @@ -100,7 +100,7 @@ dataStream.filter(new FilterFunction() { KeyBy
DataStream → KeyedStream -

Logically partitions a stream into disjoint partitions. All records with the same key are assigned to the same partition. Internally, keyBy() is implemented with hash partitioning. There are different ways to specify keys.

+

Logically partitions a stream into disjoint partitions. All records with the same key are assigned to the same partition. Internally, keyBy() is implemented with hash partitioning. There are different ways to specify keys.

This transformation returns a KeyedStream, which is, among other things, required to use keyed state.

{% highlight java %} @@ -500,7 +500,7 @@ dataStream.filter { _ != 0 } KeyBy
DataStream → KeyedStream

Logically partitions a stream into disjoint partitions, each partition containing elements of the same key. - Internally, this is implemented with hash partitioning. See keys on how to specify keys. + Internally, this is implemented with hash partitioning. See keys on how to specify keys. This transformation returns a KeyedStream.

{% highlight scala %} dataStream.keyBy("someKey") // Key by field "someKey" diff --git a/docs/dev/stream/operators/windows.zh.md b/docs/dev/stream/operators/windows.zh.md index 72269a45596d9..517f7b3386308 100644 --- a/docs/dev/stream/operators/windows.zh.md +++ b/docs/dev/stream/operators/windows.zh.md @@ -94,7 +94,7 @@ Using the `keyBy(...)` will split your infinite stream into logical keyed stream stream is not keyed. In the case of keyed streams, any attribute of your incoming events can be used as a key -(more details [here]({{ site.baseurl }}/dev/api_concepts.html#specifying-keys)). Having a keyed stream will +(more details [here]({% link dev/stream/state/state.zh.md %}#keyed-datastream)). Having a keyed stream will allow your windowed computation to be performed in parallel by multiple tasks, as each logical keyed stream can be processed independently from the rest. All elements referring to the same key will be sent to the same parallel task. diff --git a/docs/dev/stream/state/checkpointing.zh.md b/docs/dev/stream/state/checkpointing.zh.md index c940ada9de6d2..6f22d62154ab3 100644 --- a/docs/dev/stream/state/checkpointing.zh.md +++ b/docs/dev/stream/state/checkpointing.zh.md @@ -184,7 +184,7 @@ Flink 现在为没有迭代（iterations）的作业提供一致性的处理保 ## 重启策略 -Flink 支持不同的重启策略，来控制 job 万一故障时该如何重启。更多信息请阅读 [重启策略]({{ site.baseurl }}/zh/dev/restart_strategies.html)。 +Flink 支持不同的重启策略，来控制 job 万一故障时该如何重启。更多信息请阅读 [重启策略]({{ site.baseurl }}/zh/dev/task_failure_recovery.html)。 {% top %} diff --git a/docs/dev/stream/state/index.zh.md b/docs/dev/stream/state/index.zh.md index 1b5444873a0a2..ab8d9eaec2d8c 100644 --- a/docs/dev/stream/state/index.zh.md +++ b/docs/dev/stream/state/index.zh.md @@ -25,23 +25,10 @@ specific language governing permissions and limitations under the License. --> -Stateful functions and operators store data across the processing of individual elements/events, making state a critical building block for -any type of more elaborate operation. - -For example: - - - When an application searches for certain event patterns, the state will store the sequence of events encountered so far. - - When aggregating events per minute/hour/day, the state holds the pending aggregates. - - When training a machine learning model over a stream of data points, the state holds the current version of the model parameters. - - When historic data needs to be managed, the state allows efficient access to events that occurred in the past. - -Flink needs to be aware of the state in order to make state fault tolerant using [checkpoints](checkpointing.html) and to allow [savepoints]({{ site.baseurl }}/ops/state/savepoints.html) of streaming applications. - -Knowledge about the state also allows for rescaling Flink applications, meaning that Flink takes care of redistributing state across parallel instances. - -The [queryable state](queryable_state.html) feature of Flink allows you to access state from outside of Flink during runtime. - -When working with state, it might also be useful to read about [Flink's state backends]({{ site.baseurl }}/ops/state/state_backends.html). Flink provides different state backends that specify how and where state is stored. State can be located on Java's heap or off-heap. Depending on your state backend, Flink can also *manage* the state for the application, meaning Flink deals with the memory management (possibly spilling to disk if necessary) to allow applications to hold very large state. State backends can be configured without changing your application logic. +In this section you will learn about the APIs that Flink provides for writing +stateful programs. Please take a look at [Stateful Stream +Processing]({% link concepts/stateful-stream-processing.zh.md %}) +to learn about the concepts behind stateful stream processing. {% top %} diff --git a/docs/dev/stream/state/queryable_state.zh.md b/docs/dev/stream/state/queryable_state.zh.md index 1d62efda375be..6066de018273b 100644 --- a/docs/dev/stream/state/queryable_state.zh.md +++ b/docs/dev/stream/state/queryable_state.zh.md @@ -149,7 +149,7 @@ descriptor.setQueryable("query-name"); // queryable state name {% endhighlight %}
-关于依赖的更多信息, 可以参考如何 [配置 Flink 项目]({{ site.baseurl }}/zh/dev/projectsetup/dependencies.html). +关于依赖的更多信息, 可以参考如何 [配置 Flink 项目]({{ site.baseurl }}/zh/getting-started/project-setup/dependencies.html). `QueryableStateClient` 将提交你的请求到内部代理，代理会处理请求并返回结果。客户端的初始化只需要提供一个有效的 `TaskManager` 主机名 (每个 task manager 上都运行着一个 queryable state 代理)，以及代理监听的端口号。关于如何配置代理以及端口号可以参考 [Configuration Section](#configuration). diff --git a/docs/dev/stream/state/state.zh.md b/docs/dev/stream/state/state.zh.md index dad89a35fcd6c..bda020a9f8f46 100644 --- a/docs/dev/stream/state/state.zh.md +++ b/docs/dev/stream/state/state.zh.md @@ -22,51 +22,75 @@ specific language governing permissions and limitations under the License. --> -本文档主要介绍如何在 Flink 作业中使用状态 +In this section you will learn about the APIs that Flink provides for writing +stateful programs. Please take a look at [Stateful Stream +Processing]({% link concepts/stateful-stream-processing.md %}) +to learn about the concepts behind stateful stream processing. + * 目录 {:toc} -## Keyed State 与 Operator State - -Flink 中有两种基本的状态：`Keyed State` 和 `Operator State`。 - -### Keyed State - -*Keyed State* 通常和 key 相关，仅可使用在 `KeyedStream` 的方法和算子中。 +## Keyed DataStream -你可以把 Keyed State 看作分区或者共享的 Operator State, 而且每个 key 仅出现在一个分区内。 -逻辑上每个 keyed-state 和唯一元组 <算子并发实例, key> 绑定，由于每个 key 仅"属于" -算子的一个并发，因此简化为 <算子, key>。 +If you want to use keyed state, you first need to specify a key on a +`DataStream` that should be used to partition the state (and also the records +in the stream themselves). You can specify a key using `keyBy(KeySelector)` on +a `DataStream`. This will yield a `KeyedDataStream`, which then allows +operations that use keyed state. -Keyed State 会按照 *Key Group* 进行管理。Key Group 是 Flink 分发 Keyed State 的最小单元； -Key Group 的数目等于作业的最大并发数。在执行过程中，每个 keyed operator 会对应到一个或多个 Key Group +A key selector function takes a single record as input and returns the key for +that record. The key can be of any type and **must** be derived from +deterministic computations. -### Operator State +The data model of Flink is not based on key-value pairs. Therefore, you do not +need to physically pack the data set types into keys and values. Keys are +"virtual": they are defined as functions over the actual data to guide the +grouping operator. -对于 *Operator State* (或者 *non-keyed state*) 来说，每个 operator state 和一个并发实例进行绑定。 -[Kafka Connector]({{ site.baseurl }}/zh/dev/connectors/kafka.html) 是 Flink 中使用 operator state 的一个很好的示例。 -每个 Kafka 消费者的并发在 Operator State 中维护一个 topic partition 到 offset 的映射关系。 +The following example shows a key selector function that simply returns the +field of an object: -Operator State 在 Flink 作业的并发改变后，会重新分发状态，分发的策略和 Keyed State 不一样。 - -## Raw State 与 Managed State +
+
+{% highlight java %} +// some ordinary POJO +public class WC { + public String word; + public int count; -*Keyed State* 和 *Operator State* 分别有两种存在形式：*managed* and *raw*. + public String getWord() { return word; } +} +DataStream words = // [...] +KeyedStream keyed = words + .keyBy(WC::getWord); +{% endhighlight %} -*Managed State* 由 Flink 运行时控制的数据结构表示，比如内部的 hash table 或者 RocksDB。 -比如 "ValueState", "ListState" 等。Flink runtime 会对这些状态进行编码并写入 checkpoint。 +
+
+{% highlight scala %} +// some ordinary case class +case class WC(word: String, count: Int) +val words: DataStream[WC] = // [...] +val keyed = words.keyBy( _.word ) +{% endhighlight %} +
+
-*Raw State* 则保存在算子自己的数据结构中。checkpoint 的时候，Flink 并不知晓具体的内容，仅仅写入一串字节序列到 checkpoint。 +### Tuple Keys and Expression Keys +{:.no_toc} -所有 datastream 的 function 都可以使用 managed state, 但是 raw state 则只能在实现算子的时候使用。 -由于 Flink 可以在修改并发时更好的分发状态数据，并且能够更好的管理内存，因此建议使用 managed state（而不是 raw state）。 +Flink also has two alternative ways of defining keys: tuple keys and expression +keys. With this you can specify keys using tuple field indices or expressions +for selecting fields of objects. We don't recommend using these today but you +can refer to the Javadoc of DataStream to learn about them. Using a KeySelector +function is strictly superior: with Java lambdas they are easy to use and they +have potentially less overhead at runtime. -注意 如果你的 managed state 需要定制化的序列化逻辑， -为了后续的兼容性请参考 [相应指南](custom_serialization.html)，Flink 的默认序列化器不需要用户做特殊的处理。 +{% top %} -## 使用 Managed Keyed State +## 使用 Keyed State -managed keyed state 接口提供不同类型状态的访问接口，这些状态都作用于当前输入数据的 key 下。换句话说，这些状态仅可在 `KeyedStream` +keyed state 接口提供不同类型状态的访问接口，这些状态都作用于当前输入数据的 key 下。换句话说，这些状态仅可在 `KeyedStream` 上使用，可以通过 `stream.keyBy(...)` 得到 `KeyedStream`. 接下来，我们会介绍不同类型的状态，然后介绍如何使用他们。所有支持的状态类型如下所示： @@ -101,7 +125,7 @@ managed keyed state 接口提供不同类型状态的访问接口，这些状态 状态所持有值的类型，并且可能包含用户指定的函数，例如`ReduceFunction`。 根据不同的状态类型，可以创建`ValueStateDescriptor`，`ListStateDescriptor`， `ReducingStateDescriptor`，`FoldingStateDescriptor` 或 `MapStateDescriptor`。 -状态通过 `RuntimeContext` 进行访问，因此只能在 *rich functions* 中使用。请参阅[这里]({{site.baseurl}}/zh/dev/api_concepts.html#rich-functions)获取相关信息， +状态通过 `RuntimeContext` 进行访问，因此只能在 *rich functions* 中使用。请参阅[这里]({% link dev/user_defined_functions.zh.md %}#rich-functions)获取相关信息， 但是我们很快也会看到一个例子。`RichFunction` 中 `RuntimeContext` 提供如下方法： * `ValueState getState(ValueStateDescriptor)` @@ -219,7 +243,7 @@ object ExampleCountWindowAverage extends App { .print() // the printed output will be (1,4) and (1,5) - env.execute("ExampleManagedState") + env.execute("ExampleKeyedState") } {% endhighlight %}
@@ -470,9 +494,44 @@ val counts: DataStream[(String, Int)] = stream }) {% endhighlight %} -## 使用 Managed Operator State +## Operator State + +*Operator State* (or *non-keyed state*) is state that is is bound to one +parallel operator instance. The [Kafka Connector]({% link +dev/connectors/kafka.md %}) is a good motivating example for the use of +Operator State in Flink. Each parallel instance of the Kafka consumer maintains +a map of topic partitions and offsets as its Operator State. + +The Operator State interfaces support redistributing state among parallel +operator instances when the parallelism is changed. There are different schemes +for doing this redistribution. + +In a typical stateful Flink Application you don't need operators state. It is +mostly a special type of state that is used in source/sink implementations and +scenarios where you don't have a key by which state can be partitioned. + +## Broadcast State + +*Broadcast State* is a special type of *Operator State*. It was introduced to +support use cases where records of one stream need to be broadcasted to all +downstream tasks, where they are used to maintain the same state among all +subtasks. This state can then be accessed while processing records of a second +stream. As an example where broadcast state can emerge as a natural fit, one +can imagine a low-throughput stream containing a set of rules which we want to +evaluate against all elements coming from another stream. Having the above type +of use cases in mind, broadcast state differs from the rest of operator states +in that: + + 1. it has a map format, + 2. it is only available to specific operators that have as inputs a + *broadcasted* stream and a *non-broadcasted* one, and + 3. such an operator can have *multiple broadcast states* with different names. + +{% top %} + +## 使用 Operator State -用户可以通过实现 `CheckpointedFunction` 或 `ListCheckpointed` 接口来使用 managed operator state。 +用户可以通过实现 `CheckpointedFunction` 接口来使用 operator state。 #### CheckpointedFunction @@ -487,13 +546,14 @@ void initializeState(FunctionInitializationContext context) throws Exception; 进行 checkpoint 时会调用 `snapshotState()`。 用户自定义函数初始化时会调用 `initializeState()`，初始化包括第一次自定义函数初始化和从之前的 checkpoint 恢复。 因此 `initializeState()` 不仅是定义不同状态类型初始化的地方，也需要包括状态恢复的逻辑。 -当前，managed operator state 以 list 的形式存在。这些状态是一个 *可序列化* 对象的集合 `List`，彼此独立，方便在改变并发后进行状态的重新分派。 +当前 operator state 以 list 的形式存在。这些状态是一个 *可序列化* 对象的集合 `List`，彼此独立，方便在改变并发后进行状态的重新分派。 换句话说，这些对象是重新分配 non-keyed state 的最细粒度。根据状态的不同访问方式，有如下几种重新分配的模式： - **Even-split redistribution:** 每个算子都保存一个列表形式的状态集合，整个状态由所有的列表拼接而成。当作业恢复或重新分配的时候，整个状态会按照算子的并发度进行均匀分配。 比如说，算子 A 的并发读为 1，包含两个元素 `element1` 和 `element2`，当并发读增加为 2 时，`element1` 会被分到并发 0 上，`element2` 则会被分到并发 1 上。 - **Union redistribution:** 每个算子保存一个列表形式的状态集合。整个状态由所有的列表拼接而成。当作业恢复或重新分配时，每个算子都将获得所有的状态数据。 + Do not use this feature if your list may have high cardinality. Checkpoint metadata will store an offset to each list entry, which could lead to RPC framesize or out-of-memory errors. 下面的例子中的 `SinkFunction` 在 `CheckpointedFunction` 中进行数据缓存，然后统一发送到下游，这个例子演示了列表状态数据的 event-split redistribution。 diff --git a/docs/dev/table/common.zh.md b/docs/dev/table/common.zh.md index c10f2d371af34..91aab8969485d 100644 --- a/docs/dev/table/common.zh.md +++ b/docs/dev/table/common.zh.md @@ -419,7 +419,7 @@ tableEnvironment.sqlUpdate("CREATE [TEMPORARY] TABLE MyTable (...) WITH (...)") 用户可以指定一个 catalog 和数据库作为 "当前catalog" 和"当前数据库"。有了这些，那么刚刚提到的三元标识符的前两个部分就可以被省略了。如果前两部分的标识符没有指定， 那么会使用当前的 catalog 和当前数据库。用户也可以通过 Table API 或 SQL 切换当前的 catalog 和当前的数据库。 -标识符遵循 SQL 标准，因此使用时需要用反引号（`` ` ``）进行转义。此外，所有 SQL 保留关键字都必须转义。 +标识符遵循 SQL 标准，因此使用时需要用反引号（`` ` ``）进行转义。
@@ -1273,7 +1273,7 @@ val table: Table = tableEnv.fromDataStream(stream, $"age" as "myAge", $"name" as #### POJO 类型 （Java 和 Scala） -Flink 支持 POJO 类型作为复合类型。确定 POJO 类型的规则记录在[这里]({{ site.baseurl }}/zh/dev/api_concepts.html#pojos). +Flink 支持 POJO 类型作为复合类型。确定 POJO 类型的规则记录在[这里]({{ site.baseurl }}{% link dev/types_serialization.md %}#pojos). 在不指定字段名称的情况下将 POJO 类型的 `DataStream` 或 `DataSet` 转换成 `Table` 时，将使用原始 POJO 类型字段的名称。名称映射需要原始名称，并且不能按位置进行。字段可以使用别名（带有 `as` 关键字）来重命名，重新排序和投影。 diff --git a/docs/dev/table/connect.zh.md b/docs/dev/table/connect.zh.md index ba6a5f7658864..71b490c14bdbb 100644 --- a/docs/dev/table/connect.zh.md +++ b/docs/dev/table/connect.zh.md @@ -59,6 +59,8 @@ The following tables list all available connectors and formats. Their mutual com | CSV (for Kafka) | `flink-csv` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/{{site.version}}/flink-csv-{{site.version}}-sql-jar.jar) | | JSON | `flink-json` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/{{site.version}}/flink-json-{{site.version}}-sql-jar.jar) | | Apache Avro | `flink-avro` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-avro/{{site.version}}/flink-avro-{{site.version}}-sql-jar.jar) | +| Apache ORC | `flink-orc` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-orc{{site.scala_version_suffix}}/{{site.version}}/flink-orc{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) | +| Apache Parquet | `flink-parquet` | [Download](https://repo.maven.apache.org/maven2/org/apache/flink/flink-parquet{{site.scala_version_suffix}}/{{site.version}}/flink-parquet{{site.scala_version_suffix}}-{{site.version}}-jar-with-dependencies.jar) | {% else %} @@ -929,15 +931,15 @@ CREATE TABLE MyUserTable ( 'connector.hosts' = 'http://host_name:9092;http://host_name:9093', -- required: one or more Elasticsearch hosts to connect to 'connector.index' = 'myusers', -- required: Elasticsearch index. Flink supports both static index and dynamic index. - -- If you want to have a static index, this option value should be a plain string, + -- If you want to have a static index, this option value should be a plain string, -- e.g. 'myusers', all the records will be consistently written into "myusers" index. -- If you want to have a dynamic index, you can use '{field_name}' to reference a field - -- value in the record to dynamically generate a target index. You can also use + -- value in the record to dynamically generate a target index. You can also use -- '{field_name|date_format_string}' to convert a field value of TIMESTAMP/DATE/TIME type - -- into the format specified by date_format_string. The date_format_string is + -- into the format specified by date_format_string. The date_format_string is -- compatible with Java's [DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/index.html). - -- For example, if the option value is 'myusers-{log_ts|yyyy-MM-dd}', then a - -- record with log_ts field value 2020-03-27 12:25:55 will be written into + -- For example, if the option value is 'myusers-{log_ts|yyyy-MM-dd}', then a + -- record with log_ts field value 2020-03-27 12:25:55 will be written into -- "myusers-2020-03-27" index. 'connector.document-type' = 'user', -- required: Elasticsearch document type @@ -969,11 +971,11 @@ CREATE TABLE MyUserTable ( -- per bulk request -- (only MB granularity is supported) 'connector.bulk-flush.interval' = '60000', -- optional: bulk flush interval (in milliseconds) - 'connector.bulk-flush.back-off.type' = '...', -- optional: backoff strategy ("disabled" by default) + 'connector.bulk-flush.backoff.type' = '...', -- optional: backoff strategy ("disabled" by default) -- valid strategies are "disabled", "constant", -- or "exponential" - 'connector.bulk-flush.back-off.max-retries' = '3', -- optional: maximum number of retries - 'connector.bulk-flush.back-off.delay' = '30000', -- optional: delay between each backoff attempt + 'connector.bulk-flush.backoff.max-retries' = '3', -- optional: maximum number of retries + 'connector.bulk-flush.backoff.delay' = '30000', -- optional: delay between each backoff attempt -- (in milliseconds) -- optional: connection properties to be used during REST communication to Elasticsearch @@ -1199,6 +1201,27 @@ CREATE TABLE MyUserTable ( ) {% endhighlight %}
+
+{% highlight python%} +.connect( + HBase() + .version('1.4.3') # required: currently only support '1.4.3' + .table_name('hbase_table_name') # required: HBase table name + .zookeeper_quorum('localhost:2181') # required: HBase Zookeeper quorum configuration + .zookeeper_node_parent('/test') # optional: the root dir in Zookeeper for Hbae cluster. + # The default value is '/hbase' + .write_buffer_flush_max_size('10mb') # optional: writing option, determines how many size in memory of buffered + # rows to insert per round trip. This can help performance on writing to JDBC + # database. The default value is '2mb' + .write_buffer_flush_max_rows(1000) # optional: writing option, determines how many rows to insert per round trip. + # This can help performance on writing to JDBC database. No default value, + # i.e. the default flushing is not depends on the number of buffered rows. + .write_buffer_flush_interval('2s') # optional: writing option, sets a flush interval flushing buffered requesting + # if the interval passes, in milliseconds. Default value is '0s', which means + # no asynchronous flush thread will he scheduled. +) +{% endhighlight%} +
{% highlight yaml %} connector: @@ -1594,9 +1617,8 @@ CREATE TABLE MyUserTable ( 'format.type' = 'json', -- required: specify the format type 'format.fail-on-missing-field' = 'true', -- optional: flag whether to fail if a field is missing or not, -- 'false' by default - 'format.ignore-parse-errors' = 'true' -- optional: skip fields and rows with parse errors instead of failing; + 'format.ignore-parse-errors' = 'true', -- optional: skip fields and rows with parse errors instead of failing; -- fields are set to null in case of errors - -- deprecated: define the schema explicitly using JSON schema which parses to DECIMAL and TIMESTAMP. 'format.json-schema' = '{ diff --git a/docs/dev/table/index.zh.md b/docs/dev/table/index.zh.md index 8613b117830dc..561890d724281 100644 --- a/docs/dev/table/index.zh.md +++ b/docs/dev/table/index.zh.md @@ -27,7 +27,7 @@ under the License. Apache Flink 有两种关系型 API 来做流批统一处理：Table API 和 SQL。Table API 是用于 Scala 和 Java 语言的查询API，它可以用一种非常直观的方式来组合使用选取、过滤、join 等关系型算子。Flink SQL 是基于 [Apache Calcite](https://calcite.apache.org) 来实现的标准 SQL。这两种 API 中的查询对于批（DataSet）和流（DataStream）的输入有相同的语义，也会产生同样的计算结果。 -Table API 和 SQL 两种 API 是紧密集成的，以及 DataStream 和 DataSet API。你可以在这些 API 之间，以及一些基于这些 API 的库之间轻松的切换。比如，你可以先用 [CEP]({{ site.baseurl }}/zh/dev/libs/cep.html) 从 DataStream 中做模式匹配，然后用 Table API 来分析匹配的结果；或者你可以用 SQL 来扫描、过滤、聚合一个批式的表，然后再跑一个 [Gelly 图算法]({{ site.baseurl }}/zh/dev/libs/gelly) 来处理已经预处理好的数据。 +Table API 和 SQL 两种 API 是紧密集成的，以及 DataStream 和 DataSet API。你可以在这些 API 之间，以及一些基于这些 API 的库之间轻松的切换。比如，你可以先用 [CEP]({{ site.baseurl }}/zh/dev/libs/cep.html) 从 DataStream 中做模式匹配，然后用 Table API 来分析匹配的结果；或者你可以用 SQL 来扫描、过滤、聚合一个批式的表，然后再跑一个 [Gelly 图算法]({{ site.baseurl }}/zh/dev/libs/gelly/index.html) 来处理已经预处理好的数据。 **注意：Table API 和 SQL 现在还处于活跃开发阶段，还没有完全实现所有的特性。不是所有的 \[Table API，SQL\] 和 \[流，批\] 的组合都是支持的。** diff --git a/docs/dev/table/sqlClient.zh.md b/docs/dev/table/sqlClient.zh.md index c234390df014f..d2cacbee841f6 100644 --- a/docs/dev/table/sqlClient.zh.md +++ b/docs/dev/table/sqlClient.zh.md @@ -347,7 +347,7 @@ CLI commands > session environment file > defaults environment file #### 重启策略（Restart Strategies） -重启策略控制 Flink 作业失败时的重启方式。与 Flink 集群的[全局重启策略]({{ site.baseurl }}/zh/dev/restart_strategies.html)相似，更细精度的重启配置可以在环境配置文件中声明。 +重启策略控制 Flink 作业失败时的重启方式。与 Flink 集群的[全局重启策略]({{ site.baseurl }}/zh/dev/task_failure_recovery.html)相似，更细精度的重启配置可以在环境配置文件中声明。 Flink 支持以下策略： @@ -600,7 +600,7 @@ Job ID: 6f922fe5cba87406ff23ae4a7bb79044 Web interface: http://localhost:8081 {% endhighlight %} -注意 提交后，SQL 客户端不追踪正在运行的 Flink 作业状态。提交后可以关闭 CLI 进程，并且不会影响分离的查询。Flink 的[重启策略]({{ site.baseurl }}/zh/dev/restart_strategies.html)负责容错。取消查询可以用 Flink 的 web 接口、命令行或 REST API 。 +注意 提交后，SQL 客户端不追踪正在运行的 Flink 作业状态。提交后可以关闭 CLI 进程，并且不会影响分离的查询。Flink 的[重启策略]({{ site.baseurl }}/zh/dev/task_failure_recovery.html)负责容错。取消查询可以用 Flink 的 web 接口、命令行或 REST API 。 {% top %} diff --git a/docs/dev/types_serialization.zh.md b/docs/dev/types_serialization.zh.md index 72c70cc477039..5be877f9e353a 100644 --- a/docs/dev/types_serialization.zh.md +++ b/docs/dev/types_serialization.zh.md @@ -30,6 +30,206 @@ Apache Flink 以其独特的方式来处理数据类型以及序列化，这种 * This will be replaced by the TOC {:toc} +## Supported Data Types + +Flink places some restrictions on the type of elements that can be in a DataSet or DataStream. +The reason for this is that the system analyzes the types to determine +efficient execution strategies. + +There are seven different categories of data types: + +1. **Java Tuples** and **Scala Case Classes** +2. **Java POJOs** +3. **Primitive Types** +4. **Regular Classes** +5. **Values** +6. **Hadoop Writables** +7. **Special Types** + +#### Tuples and Case Classes + +
+
+ +Tuples are composite types that contain a fixed number of fields with various types. +The Java API provides classes from `Tuple1` up to `Tuple25`. Every field of a tuple +can be an arbitrary Flink type including further tuples, resulting in nested tuples. Fields of a +tuple can be accessed directly using the field's name as `tuple.f4`, or using the generic getter method +`tuple.getField(int position)`. The field indices start at 0. Note that this stands in contrast +to the Scala tuples, but it is more consistent with Java's general indexing. + +{% highlight java %} +DataStream> wordCounts = env.fromElements( + new Tuple2("hello", 1), + new Tuple2("world", 2)); + +wordCounts.map(new MapFunction, Integer>() { + @Override + public Integer map(Tuple2 value) throws Exception { + return value.f1; + } +}); + +wordCounts.keyBy(0); // also valid .keyBy("f0") + + +{% endhighlight %} + +
+
+ +Scala case classes (and Scala tuples which are a special case of case classes), are composite types that contain a fixed number of fields with various types. Tuple fields are addressed by their 1-offset names such as `_1` for the first field. Case class fields are accessed by their name. + +{% highlight scala %} +case class WordCount(word: String, count: Int) +val input = env.fromElements( + WordCount("hello", 1), + WordCount("world", 2)) // Case Class Data Set + +input.keyBy("word")// key by field expression "word" + +val input2 = env.fromElements(("hello", 1), ("world", 2)) // Tuple2 Data Set + +input2.keyBy(0, 1) // key by field positions 0 and 1 +{% endhighlight %} + +
+
+ +#### POJOs + +Java and Scala classes are treated by Flink as a special POJO data type if they fulfill the following requirements: + +- The class must be public. + +- It must have a public constructor without arguments (default constructor). + +- All fields are either public or must be accessible through getter and setter functions. For a field called `foo` the getter and setter methods must be named `getFoo()` and `setFoo()`. + +- The type of a field must be supported by a registered serializer. + +POJOs are generally represented with a `PojoTypeInfo` and serialized with the `PojoSerializer` (using [Kryo](https://github.com/EsotericSoftware/kryo) as configurable fallback). +The exception is when the POJOs are actually Avro types (Avro Specific Records) or produced as "Avro Reflect Types". +In that case the POJO's are represented by an `AvroTypeInfo` and serialized with the `AvroSerializer`. +You can also register your own custom serializer if required; see [Serialization](https://ci.apache.org/projects/flink/flink-docs-stable/dev/types_serialization.html#serialization-of-pojo-types) for further information. + +Flink analyzes the structure of POJO types, i.e., it learns about the fields of a POJO. As a result POJO types are easier to use than general types. Moreover, Flink can process POJOs more efficiently than general types. + +The following example shows a simple POJO with two public fields. + +
+
+{% highlight java %} +public class WordWithCount { + + public String word; + public int count; + + public WordWithCount() {} + + public WordWithCount(String word, int count) { + this.word = word; + this.count = count; + } +} + +DataStream wordCounts = env.fromElements( + new WordWithCount("hello", 1), + new WordWithCount("world", 2)); + +wordCounts.keyBy("word"); // key by field expression "word" + +{% endhighlight %} +
+
+{% highlight scala %} +class WordWithCount(var word: String, var count: Int) { + def this() { + this(null, -1) + } +} + +val input = env.fromElements( + new WordWithCount("hello", 1), + new WordWithCount("world", 2)) // Case Class Data Set + +input.keyBy("word")// key by field expression "word" + +{% endhighlight %} +
+
+ +#### Primitive Types + +Flink supports all Java and Scala primitive types such as `Integer`, `String`, and `Double`. + +#### General Class Types + +Flink supports most Java and Scala classes (API and custom). +Restrictions apply to classes containing fields that cannot be serialized, like file pointers, I/O streams, or other native +resources. Classes that follow the Java Beans conventions work well in general. + +All classes that are not identified as POJO types (see POJO requirements above) are handled by Flink as general class types. +Flink treats these data types as black boxes and is not able to access their content (e.g., for efficient sorting). General types are de/serialized using the serialization framework [Kryo](https://github.com/EsotericSoftware/kryo). + +#### Values + +*Value* types describe their serialization and deserialization manually. Instead of going through a +general purpose serialization framework, they provide custom code for those operations by means of +implementing the `org.apache.flinktypes.Value` interface with the methods `read` and `write`. Using +a Value type is reasonable when general purpose serialization would be highly inefficient. An +example would be a data type that implements a sparse vector of elements as an array. Knowing that +the array is mostly zero, one can use a special encoding for the non-zero elements, while the +general purpose serialization would simply write all array elements. + +The `org.apache.flinktypes.CopyableValue` interface supports manual internal cloning logic in a +similar way. + +Flink comes with pre-defined Value types that correspond to basic data types. (`ByteValue`, +`ShortValue`, `IntValue`, `LongValue`, `FloatValue`, `DoubleValue`, `StringValue`, `CharValue`, +`BooleanValue`). These Value types act as mutable variants of the basic data types: Their value can +be altered, allowing programmers to reuse objects and take pressure off the garbage collector. + + +#### Hadoop Writables + +You can use types that implement the `org.apache.hadoop.Writable` interface. The serialization logic +defined in the `write()`and `readFields()` methods will be used for serialization. + +#### Special Types + +You can use special types, including Scala's `Either`, `Option`, and `Try`. +The Java API has its own custom implementation of `Either`. +Similarly to Scala's `Either`, it represents a value of two possible types, *Left* or *Right*. +`Either` can be useful for error handling or operators that need to output two different types of records. + +#### Type Erasure & Type Inference + +*Note: This Section is only relevant for Java.* + +The Java compiler throws away much of the generic type information after compilation. This is +known as *type erasure* in Java. It means that at runtime, an instance of an object does not know +its generic type any more. For example, instances of `DataStream` and `DataStream` look the +same to the JVM. + +Flink requires type information at the time when it prepares the program for execution (when the +main method of the program is called). The Flink Java API tries to reconstruct the type information +that was thrown away in various ways and store it explicitly in the data sets and operators. You can +retrieve the type via `DataStream.getType()`. The method returns an instance of `TypeInformation`, +which is Flink's internal way of representing types. + +The type inference has its limits and needs the "cooperation" of the programmer in some cases. +Examples for that are methods that create data sets from collections, such as +`ExecutionEnvironment.fromCollection(),` where you can pass an argument that describes the type. But +also generic functions like `MapFunction` may need extra type information. + +The +{% gh_link /flink-core/src/main/java/org/apache/flink/api/java/typeutils/ResultTypeQueryable.java "ResultTypeQueryable" %} +interface can be implemented by input formats and functions to tell the API +explicitly about their return type. The *input types* that the functions are invoked with can +usually be inferred by the result types of the previous operations. + +{% top %} ## Flink 中的类型处理 diff --git a/docs/dev/user_defined_functions.zh.md b/docs/dev/user_defined_functions.zh.md index bdfbe54cee3c9..a1b52a087dc21 100644 --- a/docs/dev/user_defined_functions.zh.md +++ b/docs/dev/user_defined_functions.zh.md @@ -1,5 +1,5 @@ --- -title: 'User-Defined Functions' +title: '用户自定义函数' nav-id: user_defined_function nav-parent_id: streaming nav-pos: 4 diff --git a/docs/getting-started/index.zh.md b/docs/getting-started/index.zh.md index cb81bbc8bab04..e27119c1eb88a 100644 --- a/docs/getting-started/index.zh.md +++ b/docs/getting-started/index.zh.md @@ -27,29 +27,54 @@ specific language governing permissions and limitations under the License. --> -上手使用 Apache Flink 有很多方式，哪一个最适合你取决于你的目标和以前的经验。 +There are many ways to get started with Apache Flink. Which one is the best for +you depends on your goals and prior experience: -### 初识 Flink +* take a look at the **Docker Playgrounds** if you want to see what Flink can do, via a hands-on, + docker-based introduction to specific Flink concepts +* explore one of the **Code Walkthroughs** if you want a quick, end-to-end + introduction to one of Flink's APIs +* work your way through the **Hands-on Training** for a comprehensive, + step-by-step introduction to Flink +* use **Project Setup** if you already know the basics of Flink and want a + project template for Java or Scala, or need help setting up the dependencies -通过 **Docker Playgrounds** 提供沙箱的Flink环境，你只需花几分钟做些简单设置，就可以开始探索和使用 Flink。 +### Taking a first look at Flink -* [**Operations Playground**](./docker-playgrounds/flink-operations-playground.html) 向你展示如何使用 Flink 编写数据流应用程序。你可以体验 Flink 如何从故障中恢复应用程序，升级、提高并行度、降低并行度和监控运行的状态指标等特性。 +The **Docker Playgrounds** provide sandboxed Flink environments that are set up in just a few minutes and which allow you to explore and play with Flink. + +* The [**Operations Playground**]({% link getting-started/docker-playgrounds/flink-operations-playground.md %}) shows you how to operate streaming applications with Flink. You can experience how Flink recovers application from failures, upgrade and scale streaming applications up and down, and query application metrics. -### Flink API 入门 +### First steps with one of Flink's APIs -**代码练习**是入门的最佳方式，通过代码练习可以逐步深入理解 Flink API。 -下边的例子演示了如何使用 Flink 的代码框架开始构建一个基础的 Flink 项目，和如何逐步将其扩展为一个简单的应用程序。 +The **Code Walkthroughs** are a great way to get started quickly with a step-by-step introduction to +one of Flink's APIs. Each walkthrough provides instructions for bootstrapping a small skeleton +project, and then shows how to extend it to a simple application. - -* [**DataStream API 示例**](./walkthroughs/datastream_api.html) 展示了如何编写一个基本的 DataStream 应用程序。 DataStream API 是 Flink 的主要抽象，用于通过 Java 或 Scala 实现具有复杂时间语义的有状态数据流处理的应用程序。 +* The [**DataStream API** code walkthrough]({% link getting-started/walkthroughs/datastream_api.md %}) shows how + to implement a simple DataStream application and how to extend it to be stateful and use timers. + The DataStream API is Flink's main abstraction for implementing stateful streaming applications + with sophisticated time semantics in Java or Scala. + +* Flink's **Table API** is a relational API used for writing SQL-like queries in Java, Scala, or + Python, which are then automatically optimized, and can be executed on batch or streaming data + with identical syntax and semantics. The [Table API code walkthrough for Java and Scala]({% link + getting-started/walkthroughs/table_api.md %}) shows how to implement a simple Table API query on a + batch source and how to evolve it into a continuous query on a streaming source. There's also a + similar [code walkthrough for the Python Table API]({% link + getting-started/walkthroughs/python_table_api.md %}). + +### Taking a Deep Dive with the Hands-on Training -* [**Table API 示例**](./walkthroughs/table_api.html) 演示了如何在批处中使用简单的 Table API 进行查询，以及如何将其扩展为流处理中的查询。Table API 是 Flink 的语言嵌入式关系 API，用于在 Java 或 Scala 中编写类 SQL 的查询，这些查询会自动进行优化。Table API 查询可以使用一致的语法和语义同时在批处理或流数据上运行。 +The [**Hands-on Training**]({% link training/index.md %}) is a self-paced training course with +a set of lessons and hands-on exercises. This step-by-step introduction to Flink focuses +on learning how to use the DataStream API to meet the needs of common, real-world use cases, +and provides a complete introduction to the fundamental concepts: parallel dataflows, +stateful stream processing, event time and watermarking, and fault tolerance via state snapshots. + + + plugins + + dir + + + true + flink-${project.version} + + + + + + ../flink-metrics/flink-metrics-jmx/target/flink-metrics-jmx_${scala.binary.version}-${project.version}.jar + plugins/metrics_jmx/ + flink-metrics-jmx-${project.version}.jar + 0644 + + + + From fd37030593a00a30316deacd9c4856e93b61b6a8 Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Tue, 19 May 2020 11:00:57 +0200 Subject: [PATCH 046/773] [FLINK-17809][dist] Quote classpath and FLINK_CONF_DIR --- flink-dist/src/main/flink-bin/bin/config.sh | 4 ++-- flink-dist/src/main/flink-bin/bin/taskmanager.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh index fe69eada5beda..01d6308ddf37e 100755 --- a/flink-dist/src/main/flink-bin/bin/config.sh +++ b/flink-dist/src/main/flink-bin/bin/config.sh @@ -478,9 +478,9 @@ runBashJavaUtilsCmd() { local conf_dir=$2 local class_path=$3 local dynamic_args=${@:4} - class_path=`manglePathList ${class_path}` + class_path=`manglePathList "${class_path}"` - local output=`${JAVA_RUN} -classpath ${class_path} org.apache.flink.runtime.util.bash.BashJavaUtils ${cmd} --configDir ${conf_dir} $dynamic_args 2>&1 | tail -n 1000` + local output=`${JAVA_RUN} -classpath "${class_path}" org.apache.flink.runtime.util.bash.BashJavaUtils ${cmd} --configDir "${conf_dir}" $dynamic_args 2>&1 | tail -n 1000` if [[ $? -ne 0 ]]; then echo "[ERROR] Cannot run BashJavaUtils to execute command ${cmd}." 1>&2 # Print the output in case the user redirect the log to console. diff --git a/flink-dist/src/main/flink-bin/bin/taskmanager.sh b/flink-dist/src/main/flink-bin/bin/taskmanager.sh index c6e3aab9d912a..6c9532688c20a 100755 --- a/flink-dist/src/main/flink-bin/bin/taskmanager.sh +++ b/flink-dist/src/main/flink-bin/bin/taskmanager.sh @@ -48,7 +48,7 @@ if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then # Startup parameters - java_utils_output=$(runBashJavaUtilsCmd GET_TM_RESOURCE_PARAMS ${FLINK_CONF_DIR} $FLINK_BIN_DIR/bash-java-utils.jar:$(findFlinkDistJar) "${ARGS[@]}") + java_utils_output=$(runBashJavaUtilsCmd GET_TM_RESOURCE_PARAMS "${FLINK_CONF_DIR}" "$FLINK_BIN_DIR/bash-java-utils.jar:$(findFlinkDistJar)" "${ARGS[@]}") logging_output=$(extractLoggingOutputs "${java_utils_output}") params_output=$(extractExecutionResults "${java_utils_output}" 2) From cf29c2c9c21eba72535a5eb86d6f28de2803c0ee Mon Sep 17 00:00:00 2001 From: Gyula Fora Date: Wed, 13 May 2020 11:23:50 +0200 Subject: [PATCH 047/773] [FLINK-17619] Disable commit on checkpoints if no group.id was specified for Kafka table source Closes #12250 --- .../kafka/FlinkKafkaConsumerBase.java | 5 +++++ .../kafka/KafkaTableSourceBase.java | 1 + .../KafkaTableSourceSinkFactoryTestBase.java | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index f9f835a0b7705..84057b0847d79 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -1114,6 +1114,11 @@ LinkedMap getPendingOffsetsToCommit() { return pendingOffsetsToCommit; } + @VisibleForTesting + boolean getEnableCommitOnCheckpoints() { + return enableCommitOnCheckpoints; + } + /** * Creates state serializer for kafka topic partition to offset tuple. * Using of the explicit state serializer with KryoSerializer is needed because otherwise diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java index d6195e6b1a09d..ef4051a78d4f8 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java @@ -291,6 +291,7 @@ protected FlinkKafkaConsumerBase getKafkaConsumer( kafkaConsumer.setStartFromTimestamp(startupTimestampMillis); break; } + kafkaConsumer.setCommitOffsetsOnCheckpoints(properties.getProperty("group.id") != null); return kafkaConsumer; } diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java index d8eb0112251ea..218e55d3ec310 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java @@ -67,6 +67,7 @@ import java.util.Properties; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -169,6 +170,26 @@ public void testTableSource() { final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock(); actualKafkaSource.getDataStream(mock); assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass())); + assertTrue(((FlinkKafkaConsumerBase) mock.sourceFunction).getEnableCommitOnCheckpoints()); + + Properties propsWithoutGroupId = new Properties(); + propsWithoutGroupId.setProperty("bootstrap.servers", "dummy"); + + final KafkaTableSourceBase sourceWithoutGroupId = getExpectedKafkaTableSource( + schema, + Optional.of(PROC_TIME), + rowtimeAttributeDescriptors, + fieldMapping, + TOPIC, + propsWithoutGroupId, + deserializationSchema, + StartupMode.LATEST, + new HashMap<>(), + 0L); + + sourceWithoutGroupId.getDataStream(mock); + assertTrue(mock.sourceFunction instanceof FlinkKafkaConsumerBase); + assertFalse(((FlinkKafkaConsumerBase) mock.sourceFunction).getEnableCommitOnCheckpoints()); } @Test From 4fba374386232de92fa4acddeb2205471cc17cfa Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Mon, 18 May 2020 09:38:49 +0200 Subject: [PATCH 048/773] [FLINK-17763][dist] Properly handle log properties and spaces in scala-shell.sh --- flink-scala-shell/start-script/start-scala-shell.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-scala-shell/start-script/start-scala-shell.sh b/flink-scala-shell/start-script/start-scala-shell.sh index e083008ff8235..bc1f9b53b062e 100644 --- a/flink-scala-shell/start-script/start-scala-shell.sh +++ b/flink-scala-shell/start-script/start-scala-shell.sh @@ -93,13 +93,13 @@ then FLINK_CLASSPATH=$FLINK_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR fi -log_setting="-Dlog.file="$LOG" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/$LOG4J_CONFIG -Dlog4j.configurationFile=file:"$FLINK_CONF_DIR"/$LOG4J_CONFIG -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/$LOGBACK_CONFIG" +log_setting=("-Dlog.file=$LOG" "-Dlog4j.configuration=file:$FLINK_CONF_DIR/$LOG4J_CONFIG" "-Dlog4j.configurationFile=file:$FLINK_CONF_DIR/$LOG4J_CONFIG" "-Dlogback.configurationFile=file:$FLINK_CONF_DIR/$LOGBACK_CONFIG") if ${EXTERNAL_LIB_FOUND} then - $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" "$log_setting" org.apache.flink.api.scala.FlinkShell $@ --addclasspath "$EXT_CLASSPATH" + $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" "${log_setting[@]}" org.apache.flink.api.scala.FlinkShell $@ --addclasspath "$EXT_CLASSPATH" else - $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" "$log_setting" org.apache.flink.api.scala.FlinkShell $@ + $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" "${log_setting[@]}" org.apache.flink.api.scala.FlinkShell $@ fi #restore echo From 9fedade3f8ec4b3dad0e9d7d0a6751fc2c66a121 Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Thu, 7 May 2020 09:33:25 +0200 Subject: [PATCH 049/773] [FLINK-17547][task][hotfix] Improve error handling 1 catch one more invalid input in DataOutputSerializer.write 2 more informative error messages --- .../org/apache/flink/core/memory/DataOutputSerializer.java | 4 ++-- .../org/apache/flink/core/memory/HybridMemorySegment.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java b/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java index 85fd76723b393..1255fc34a1d09 100644 --- a/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/core/memory/DataOutputSerializer.java @@ -157,8 +157,8 @@ public void write(byte[] b, int off, int len) throws IOException { @Override public void write(MemorySegment segment, int off, int len) throws IOException { - if (len < 0 || off > segment.size() - len) { - throw new ArrayIndexOutOfBoundsException(); + if (len < 0 || off < 0 || off > segment.size() - len) { + throw new IndexOutOfBoundsException(String.format("offset: %d, length: %d, size: %d", off, len, segment.size())); } if (this.position > this.buffer.length - len) { resize(len); diff --git a/flink-core/src/main/java/org/apache/flink/core/memory/HybridMemorySegment.java b/flink-core/src/main/java/org/apache/flink/core/memory/HybridMemorySegment.java index fb7a4baa87035..53e8cfdf2c575 100644 --- a/flink-core/src/main/java/org/apache/flink/core/memory/HybridMemorySegment.java +++ b/flink-core/src/main/java/org/apache/flink/core/memory/HybridMemorySegment.java @@ -195,8 +195,7 @@ else if (address > addressLimit) { throw new IllegalStateException("segment has been freed"); } else { - // index is in fact invalid - throw new IndexOutOfBoundsException(); + throw new IndexOutOfBoundsException(String.format("pos: %d, length: %d, index: %d, offset: %d", pos, length, index, offset)); } } From 39f5f1b0f09c37400ba113fdf33f90a832de5f0d Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Wed, 6 May 2020 17:54:05 +0200 Subject: [PATCH 050/773] [FLINK-17547][task][hotfix] Extract NonSpanningWrapper from SpillingAdaptiveSpanningRecordDeserializer (static inner class) As it is, no logical changes. --- .../api/serialization/NonSpanningWrapper.java | 296 ++++++++++++++++++ ...ingAdaptiveSpanningRecordDeserializer.java | 271 ---------------- 2 files changed, 296 insertions(+), 271 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java new file mode 100644 index 0000000000000..bab50fafd94aa --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.network.api.serialization; + +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.io.UTFDataFormatException; +import java.util.Optional; + +final class NonSpanningWrapper implements DataInputView { + + MemorySegment segment; + + private int limit; + + int position; + + private byte[] utfByteBuffer; // reusable byte buffer for utf-8 decoding + private char[] utfCharBuffer; // reusable char buffer for utf-8 decoding + + int remaining() { + return this.limit - this.position; + } + + void clear() { + this.segment = null; + this.limit = 0; + this.position = 0; + } + + void initializeFromMemorySegment(MemorySegment seg, int position, int leftOverLimit) { + this.segment = seg; + this.position = position; + this.limit = leftOverLimit; + } + + Optional getUnconsumedSegment() { + if (remaining() == 0) { + return Optional.empty(); + } + MemorySegment target = MemorySegmentFactory.allocateUnpooledSegment(remaining()); + segment.copyTo(position, target, 0, remaining()); + return Optional.of(target); + } + + // ------------------------------------------------------------------------------------------------------------- + // DataInput specific methods + // ------------------------------------------------------------------------------------------------------------- + + @Override + public final void readFully(byte[] b) throws IOException { + readFully(b, 0, b.length); + } + + @Override + public final void readFully(byte[] b, int off, int len) throws IOException { + if (off < 0 || len < 0 || off + len > b.length) { + throw new IndexOutOfBoundsException(); + } + + this.segment.get(this.position, b, off, len); + this.position += len; + } + + @Override + public final boolean readBoolean() throws IOException { + return readByte() == 1; + } + + @Override + public final byte readByte() throws IOException { + return this.segment.get(this.position++); + } + + @Override + public final int readUnsignedByte() throws IOException { + return readByte() & 0xff; + } + + @Override + public final short readShort() throws IOException { + final short v = this.segment.getShortBigEndian(this.position); + this.position += 2; + return v; + } + + @Override + public final int readUnsignedShort() throws IOException { + final int v = this.segment.getShortBigEndian(this.position) & 0xffff; + this.position += 2; + return v; + } + + @Override + public final char readChar() throws IOException { + final char v = this.segment.getCharBigEndian(this.position); + this.position += 2; + return v; + } + + @Override + public final int readInt() throws IOException { + final int v = this.segment.getIntBigEndian(this.position); + this.position += 4; + return v; + } + + @Override + public final long readLong() throws IOException { + final long v = this.segment.getLongBigEndian(this.position); + this.position += 8; + return v; + } + + @Override + public final float readFloat() throws IOException { + return Float.intBitsToFloat(readInt()); + } + + @Override + public final double readDouble() throws IOException { + return Double.longBitsToDouble(readLong()); + } + + @Override + public final String readLine() throws IOException { + final StringBuilder bld = new StringBuilder(32); + + try { + int b; + while ((b = readUnsignedByte()) != '\n') { + if (b != '\r') { + bld.append((char) b); + } + } + } + catch (EOFException ignored) {} + + if (bld.length() == 0) { + return null; + } + + // trim a trailing carriage return + int len = bld.length(); + if (len > 0 && bld.charAt(len - 1) == '\r') { + bld.setLength(len - 1); + } + return bld.toString(); + } + + @Override + public final String readUTF() throws IOException { + final int utflen = readUnsignedShort(); + + final byte[] bytearr; + final char[] chararr; + + if (this.utfByteBuffer == null || this.utfByteBuffer.length < utflen) { + bytearr = new byte[utflen]; + this.utfByteBuffer = bytearr; + } else { + bytearr = this.utfByteBuffer; + } + if (this.utfCharBuffer == null || this.utfCharBuffer.length < utflen) { + chararr = new char[utflen]; + this.utfCharBuffer = chararr; + } else { + chararr = this.utfCharBuffer; + } + + int c, char2, char3; + int count = 0; + int chararrCount = 0; + + readFully(bytearr, 0, utflen); + + while (count < utflen) { + c = (int) bytearr[count] & 0xff; + if (c > 127) { + break; + } + count++; + chararr[chararrCount++] = (char) c; + } + + while (count < utflen) { + c = (int) bytearr[count] & 0xff; + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + count++; + chararr[chararrCount++] = (char) c; + break; + case 12: + case 13: + count += 2; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = (int) bytearr[count - 1]; + if ((char2 & 0xC0) != 0x80) { + throw new UTFDataFormatException("malformed input around byte " + count); + } + chararr[chararrCount++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); + break; + case 14: + count += 3; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = (int) bytearr[count - 2]; + char3 = (int) bytearr[count - 1]; + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { + throw new UTFDataFormatException("malformed input around byte " + (count - 1)); + } + chararr[chararrCount++] = (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); + break; + default: + throw new UTFDataFormatException("malformed input around byte " + count); + } + } + // The number of chars produced may be less than utflen + return new String(chararr, 0, chararrCount); + } + + @Override + public final int skipBytes(int n) throws IOException { + if (n < 0) { + throw new IllegalArgumentException(); + } + + int toSkip = Math.min(n, remaining()); + this.position += toSkip; + return toSkip; + } + + @Override + public void skipBytesToRead(int numBytes) throws IOException { + int skippedBytes = skipBytes(numBytes); + + if (skippedBytes < numBytes){ + throw new EOFException("Could not skip " + numBytes + " bytes."); + } + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (b == null){ + throw new NullPointerException("Byte array b cannot be null."); + } + + if (off < 0){ + throw new IllegalArgumentException("The offset off cannot be negative."); + } + + if (len < 0){ + throw new IllegalArgumentException("The length len cannot be negative."); + } + + int toRead = Math.min(len, remaining()); + this.segment.get(this.position, b, off, toRead); + this.position += toRead; + + return toRead; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java index 346bdfc766239..5003e78104997 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java @@ -32,12 +32,10 @@ import org.apache.flink.util.StringUtils; import java.io.BufferedInputStream; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.RandomAccessFile; -import java.io.UTFDataFormatException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; @@ -184,275 +182,6 @@ public boolean hasUnfinishedData() { // ----------------------------------------------------------------------------------------------------------------- - private static final class NonSpanningWrapper implements DataInputView { - - private MemorySegment segment; - - private int limit; - - private int position; - - private byte[] utfByteBuffer; // reusable byte buffer for utf-8 decoding - private char[] utfCharBuffer; // reusable char buffer for utf-8 decoding - - int remaining() { - return this.limit - this.position; - } - - void clear() { - this.segment = null; - this.limit = 0; - this.position = 0; - } - - void initializeFromMemorySegment(MemorySegment seg, int position, int leftOverLimit) { - this.segment = seg; - this.position = position; - this.limit = leftOverLimit; - } - - Optional getUnconsumedSegment() { - if (remaining() == 0) { - return Optional.empty(); - } - MemorySegment target = MemorySegmentFactory.allocateUnpooledSegment(remaining()); - segment.copyTo(position, target, 0, remaining()); - return Optional.of(target); - } - - // ------------------------------------------------------------------------------------------------------------- - // DataInput specific methods - // ------------------------------------------------------------------------------------------------------------- - - @Override - public final void readFully(byte[] b) throws IOException { - readFully(b, 0, b.length); - } - - @Override - public final void readFully(byte[] b, int off, int len) throws IOException { - if (off < 0 || len < 0 || off + len > b.length) { - throw new IndexOutOfBoundsException(); - } - - this.segment.get(this.position, b, off, len); - this.position += len; - } - - @Override - public final boolean readBoolean() throws IOException { - return readByte() == 1; - } - - @Override - public final byte readByte() throws IOException { - return this.segment.get(this.position++); - } - - @Override - public final int readUnsignedByte() throws IOException { - return readByte() & 0xff; - } - - @Override - public final short readShort() throws IOException { - final short v = this.segment.getShortBigEndian(this.position); - this.position += 2; - return v; - } - - @Override - public final int readUnsignedShort() throws IOException { - final int v = this.segment.getShortBigEndian(this.position) & 0xffff; - this.position += 2; - return v; - } - - @Override - public final char readChar() throws IOException { - final char v = this.segment.getCharBigEndian(this.position); - this.position += 2; - return v; - } - - @Override - public final int readInt() throws IOException { - final int v = this.segment.getIntBigEndian(this.position); - this.position += 4; - return v; - } - - @Override - public final long readLong() throws IOException { - final long v = this.segment.getLongBigEndian(this.position); - this.position += 8; - return v; - } - - @Override - public final float readFloat() throws IOException { - return Float.intBitsToFloat(readInt()); - } - - @Override - public final double readDouble() throws IOException { - return Double.longBitsToDouble(readLong()); - } - - @Override - public final String readLine() throws IOException { - final StringBuilder bld = new StringBuilder(32); - - try { - int b; - while ((b = readUnsignedByte()) != '\n') { - if (b != '\r') { - bld.append((char) b); - } - } - } - catch (EOFException ignored) {} - - if (bld.length() == 0) { - return null; - } - - // trim a trailing carriage return - int len = bld.length(); - if (len > 0 && bld.charAt(len - 1) == '\r') { - bld.setLength(len - 1); - } - return bld.toString(); - } - - @Override - public final String readUTF() throws IOException { - final int utflen = readUnsignedShort(); - - final byte[] bytearr; - final char[] chararr; - - if (this.utfByteBuffer == null || this.utfByteBuffer.length < utflen) { - bytearr = new byte[utflen]; - this.utfByteBuffer = bytearr; - } else { - bytearr = this.utfByteBuffer; - } - if (this.utfCharBuffer == null || this.utfCharBuffer.length < utflen) { - chararr = new char[utflen]; - this.utfCharBuffer = chararr; - } else { - chararr = this.utfCharBuffer; - } - - int c, char2, char3; - int count = 0; - int chararrCount = 0; - - readFully(bytearr, 0, utflen); - - while (count < utflen) { - c = (int) bytearr[count] & 0xff; - if (c > 127) { - break; - } - count++; - chararr[chararrCount++] = (char) c; - } - - while (count < utflen) { - c = (int) bytearr[count] & 0xff; - switch (c >> 4) { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - count++; - chararr[chararrCount++] = (char) c; - break; - case 12: - case 13: - count += 2; - if (count > utflen) { - throw new UTFDataFormatException("malformed input: partial character at end"); - } - char2 = (int) bytearr[count - 1]; - if ((char2 & 0xC0) != 0x80) { - throw new UTFDataFormatException("malformed input around byte " + count); - } - chararr[chararrCount++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); - break; - case 14: - count += 3; - if (count > utflen) { - throw new UTFDataFormatException("malformed input: partial character at end"); - } - char2 = (int) bytearr[count - 2]; - char3 = (int) bytearr[count - 1]; - if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { - throw new UTFDataFormatException("malformed input around byte " + (count - 1)); - } - chararr[chararrCount++] = (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); - break; - default: - throw new UTFDataFormatException("malformed input around byte " + count); - } - } - // The number of chars produced may be less than utflen - return new String(chararr, 0, chararrCount); - } - - @Override - public final int skipBytes(int n) throws IOException { - if (n < 0) { - throw new IllegalArgumentException(); - } - - int toSkip = Math.min(n, remaining()); - this.position += toSkip; - return toSkip; - } - - @Override - public void skipBytesToRead(int numBytes) throws IOException { - int skippedBytes = skipBytes(numBytes); - - if (skippedBytes < numBytes){ - throw new EOFException("Could not skip " + numBytes + " bytes."); - } - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (b == null){ - throw new NullPointerException("Byte array b cannot be null."); - } - - if (off < 0){ - throw new IllegalArgumentException("The offset off cannot be negative."); - } - - if (len < 0){ - throw new IllegalArgumentException("The length len cannot be negative."); - } - - int toRead = Math.min(len, remaining()); - this.segment.get(this.position, b, off, toRead); - this.position += toRead; - - return toRead; - } - - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - } - // ----------------------------------------------------------------------------------------------------------------- private static final class SpanningWrapper { From 5fd01eacd2e49673b0ff1532d0798844639569de Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Wed, 6 May 2020 17:55:48 +0200 Subject: [PATCH 051/773] [FLINK-17547][task][hotfix] Extract SpanningWrapper from SpillingAdaptiveSpanningRecordDeserializer (static inner class). As it is, no logical changes. --- .../api/serialization/SpanningWrapper.java | 297 ++++++++++++++++++ ...ingAdaptiveSpanningRecordDeserializer.java | 278 ---------------- 2 files changed, 297 insertions(+), 278 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java new file mode 100644 index 0000000000000..e59363f7c2865 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.network.api.serialization; + +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.util.FileUtils; +import org.apache.flink.util.StringUtils; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.Arrays; +import java.util.Optional; +import java.util.Random; + +final class SpanningWrapper { + + private static final int THRESHOLD_FOR_SPILLING = 5 * 1024 * 1024; // 5 MiBytes + + private final byte[] initialBuffer = new byte[1024]; + + private final String[] tempDirs; + + private final Random rnd = new Random(); + + private final DataInputDeserializer serializationReadBuffer; + + private final ByteBuffer lengthBuffer; + + private FileChannel spillingChannel; + + private byte[] buffer; + + private int recordLength; + + private int accumulatedRecordBytes; + + private MemorySegment leftOverData; + + private int leftOverStart; + + private int leftOverLimit; + + private File spillFile; + + private DataInputViewStreamWrapper spillFileReader; + + public SpanningWrapper(String[] tempDirs) { + this.tempDirs = tempDirs; + + this.lengthBuffer = ByteBuffer.allocate(4); + this.lengthBuffer.order(ByteOrder.BIG_ENDIAN); + + this.recordLength = -1; + + this.serializationReadBuffer = new DataInputDeserializer(); + this.buffer = initialBuffer; + } + + void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { + // set the length and copy what is available to the buffer + this.recordLength = nextRecordLength; + + final int numBytesChunk = partial.remaining(); + + if (nextRecordLength > THRESHOLD_FOR_SPILLING) { + // create a spilling channel and put the data there + this.spillingChannel = createSpillingChannel(); + + ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); + FileUtils.writeCompletely(this.spillingChannel, toWrite); + } + else { + // collect in memory + ensureBufferCapacity(nextRecordLength); + partial.segment.get(partial.position, buffer, 0, numBytesChunk); + } + + this.accumulatedRecordBytes = numBytesChunk; + } + + void initializeWithPartialLength(NonSpanningWrapper partial) throws IOException { + // copy what we have to the length buffer + partial.segment.get(partial.position, this.lengthBuffer, partial.remaining()); + } + + void addNextChunkFromMemorySegment(MemorySegment segment, int offset, int numBytes) throws IOException { + int segmentPosition = offset; + int segmentRemaining = numBytes; + // check where to go. if we have a partial length, we need to complete it first + if (this.lengthBuffer.position() > 0) { + int toPut = Math.min(this.lengthBuffer.remaining(), segmentRemaining); + segment.get(segmentPosition, this.lengthBuffer, toPut); + // did we complete the length? + if (this.lengthBuffer.hasRemaining()) { + return; + } else { + this.recordLength = this.lengthBuffer.getInt(0); + + this.lengthBuffer.clear(); + segmentPosition += toPut; + segmentRemaining -= toPut; + if (this.recordLength > THRESHOLD_FOR_SPILLING) { + this.spillingChannel = createSpillingChannel(); + } else { + ensureBufferCapacity(this.recordLength); + } + } + } + + // copy as much as we need or can for this next spanning record + int needed = this.recordLength - this.accumulatedRecordBytes; + int toCopy = Math.min(needed, segmentRemaining); + + if (spillingChannel != null) { + // spill to file + ByteBuffer toWrite = segment.wrap(segmentPosition, toCopy); + FileUtils.writeCompletely(this.spillingChannel, toWrite); + } else { + segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy); + } + + this.accumulatedRecordBytes += toCopy; + + if (toCopy < segmentRemaining) { + // there is more data in the segment + this.leftOverData = segment; + this.leftOverStart = segmentPosition + toCopy; + this.leftOverLimit = numBytes + offset; + } + + if (accumulatedRecordBytes == recordLength) { + // we have the full record + if (spillingChannel == null) { + this.serializationReadBuffer.setBuffer(buffer, 0, recordLength); + } + else { + spillingChannel.close(); + + BufferedInputStream inStream = new BufferedInputStream(new FileInputStream(spillFile), 2 * 1024 * 1024); + this.spillFileReader = new DataInputViewStreamWrapper(inStream); + } + } + } + + Optional getUnconsumedSegment() throws IOException { + // for the case of only partial length, no data + final int position = lengthBuffer.position(); + if (position > 0) { + MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(position); + lengthBuffer.position(0); + segment.put(0, lengthBuffer, position); + return Optional.of(segment); + } + + // for the case of full length, partial data in buffer + if (recordLength > THRESHOLD_FOR_SPILLING) { + throw new UnsupportedOperationException("Unaligned checkpoint currently do not support spilled " + + "records."); + } else if (recordLength != -1) { + int leftOverSize = leftOverLimit - leftOverStart; + int unconsumedSize = Integer.BYTES + accumulatedRecordBytes + leftOverSize; + DataOutputSerializer serializer = new DataOutputSerializer(unconsumedSize); + serializer.writeInt(recordLength); + serializer.write(buffer, 0, accumulatedRecordBytes); + if (leftOverData != null) { + serializer.write(leftOverData, leftOverStart, leftOverSize); + } + MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(unconsumedSize); + segment.put(0, serializer.getSharedBuffer(), 0, segment.size()); + return Optional.of(segment); + } + + // for the case of no remaining partial length or data + return Optional.empty(); + } + + void moveRemainderToNonSpanningDeserializer(NonSpanningWrapper deserializer) { + deserializer.clear(); + + if (leftOverData != null) { + deserializer.initializeFromMemorySegment(leftOverData, leftOverStart, leftOverLimit); + } + } + + boolean hasFullRecord() { + return this.recordLength >= 0 && this.accumulatedRecordBytes >= this.recordLength; + } + + int getNumGatheredBytes() { + return this.accumulatedRecordBytes + (this.recordLength >= 0 ? 4 : lengthBuffer.position()); + } + + public void clear() { + this.buffer = initialBuffer; + this.serializationReadBuffer.releaseArrays(); + + this.recordLength = -1; + this.lengthBuffer.clear(); + this.leftOverData = null; + this.leftOverStart = 0; + this.leftOverLimit = 0; + this.accumulatedRecordBytes = 0; + + if (spillingChannel != null) { + try { + spillingChannel.close(); + } + catch (Throwable t) { + // ignore + } + spillingChannel = null; + } + if (spillFileReader != null) { + try { + spillFileReader.close(); + } + catch (Throwable t) { + // ignore + } + spillFileReader = null; + } + if (spillFile != null) { + spillFile.delete(); + spillFile = null; + } + } + + public DataInputView getInputView() { + if (spillFileReader == null) { + return serializationReadBuffer; + } + else { + return spillFileReader; + } + } + + private void ensureBufferCapacity(int minLength) { + if (buffer.length < minLength) { + byte[] newBuffer = new byte[Math.max(minLength, buffer.length * 2)]; + System.arraycopy(buffer, 0, newBuffer, 0, accumulatedRecordBytes); + buffer = newBuffer; + } + } + + @SuppressWarnings("resource") + private FileChannel createSpillingChannel() throws IOException { + if (spillFile != null) { + throw new IllegalStateException("Spilling file already exists."); + } + + // try to find a unique file name for the spilling channel + int maxAttempts = 10; + for (int attempt = 0; attempt < maxAttempts; attempt++) { + String directory = tempDirs[rnd.nextInt(tempDirs.length)]; + spillFile = new File(directory, randomString(rnd) + ".inputchannel"); + if (spillFile.createNewFile()) { + return new RandomAccessFile(spillFile, "rw").getChannel(); + } + } + + throw new IOException( + "Could not find a unique file channel name in '" + Arrays.toString(tempDirs) + + "' for spilling large records during deserialization."); + } + + private static String randomString(Random random) { + final byte[] bytes = new byte[20]; + random.nextBytes(bytes); + return StringUtils.byteToHexString(bytes); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java index 5003e78104997..f20fbc92c73b8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java @@ -19,29 +19,13 @@ package org.apache.flink.runtime.io.network.api.serialization; import org.apache.flink.core.io.IOReadableWritable; -import org.apache.flink.core.memory.DataInputDeserializer; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataInputViewStreamWrapper; -import org.apache.flink.core.memory.DataOutputSerializer; import org.apache.flink.core.memory.MemorySegment; -import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; -import org.apache.flink.util.FileUtils; -import org.apache.flink.util.StringUtils; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; -import java.util.Arrays; import java.util.Optional; -import java.util.Random; /** * @param The type of the record to be deserialized. @@ -54,8 +38,6 @@ public class SpillingAdaptiveSpanningRecordDeserializer 0 || this.spanningWrapper.getNumGatheredBytes() > 0; } - - // ----------------------------------------------------------------------------------------------------------------- - - // ----------------------------------------------------------------------------------------------------------------- - - private static final class SpanningWrapper { - - private final byte[] initialBuffer = new byte[1024]; - - private final String[] tempDirs; - - private final Random rnd = new Random(); - - private final DataInputDeserializer serializationReadBuffer; - - private final ByteBuffer lengthBuffer; - - private FileChannel spillingChannel; - - private byte[] buffer; - - private int recordLength; - - private int accumulatedRecordBytes; - - private MemorySegment leftOverData; - - private int leftOverStart; - - private int leftOverLimit; - - private File spillFile; - - private DataInputViewStreamWrapper spillFileReader; - - public SpanningWrapper(String[] tempDirs) { - this.tempDirs = tempDirs; - - this.lengthBuffer = ByteBuffer.allocate(4); - this.lengthBuffer.order(ByteOrder.BIG_ENDIAN); - - this.recordLength = -1; - - this.serializationReadBuffer = new DataInputDeserializer(); - this.buffer = initialBuffer; - } - - private void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { - // set the length and copy what is available to the buffer - this.recordLength = nextRecordLength; - - final int numBytesChunk = partial.remaining(); - - if (nextRecordLength > THRESHOLD_FOR_SPILLING) { - // create a spilling channel and put the data there - this.spillingChannel = createSpillingChannel(); - - ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); - FileUtils.writeCompletely(this.spillingChannel, toWrite); - } - else { - // collect in memory - ensureBufferCapacity(nextRecordLength); - partial.segment.get(partial.position, buffer, 0, numBytesChunk); - } - - this.accumulatedRecordBytes = numBytesChunk; - } - - private void initializeWithPartialLength(NonSpanningWrapper partial) throws IOException { - // copy what we have to the length buffer - partial.segment.get(partial.position, this.lengthBuffer, partial.remaining()); - } - - private void addNextChunkFromMemorySegment(MemorySegment segment, int offset, int numBytes) throws IOException { - int segmentPosition = offset; - int segmentRemaining = numBytes; - // check where to go. if we have a partial length, we need to complete it first - if (this.lengthBuffer.position() > 0) { - int toPut = Math.min(this.lengthBuffer.remaining(), segmentRemaining); - segment.get(segmentPosition, this.lengthBuffer, toPut); - // did we complete the length? - if (this.lengthBuffer.hasRemaining()) { - return; - } else { - this.recordLength = this.lengthBuffer.getInt(0); - - this.lengthBuffer.clear(); - segmentPosition += toPut; - segmentRemaining -= toPut; - if (this.recordLength > THRESHOLD_FOR_SPILLING) { - this.spillingChannel = createSpillingChannel(); - } else { - ensureBufferCapacity(this.recordLength); - } - } - } - - // copy as much as we need or can for this next spanning record - int needed = this.recordLength - this.accumulatedRecordBytes; - int toCopy = Math.min(needed, segmentRemaining); - - if (spillingChannel != null) { - // spill to file - ByteBuffer toWrite = segment.wrap(segmentPosition, toCopy); - FileUtils.writeCompletely(this.spillingChannel, toWrite); - } else { - segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy); - } - - this.accumulatedRecordBytes += toCopy; - - if (toCopy < segmentRemaining) { - // there is more data in the segment - this.leftOverData = segment; - this.leftOverStart = segmentPosition + toCopy; - this.leftOverLimit = numBytes + offset; - } - - if (accumulatedRecordBytes == recordLength) { - // we have the full record - if (spillingChannel == null) { - this.serializationReadBuffer.setBuffer(buffer, 0, recordLength); - } - else { - spillingChannel.close(); - - BufferedInputStream inStream = new BufferedInputStream(new FileInputStream(spillFile), 2 * 1024 * 1024); - this.spillFileReader = new DataInputViewStreamWrapper(inStream); - } - } - } - - Optional getUnconsumedSegment() throws IOException { - // for the case of only partial length, no data - final int position = lengthBuffer.position(); - if (position > 0) { - MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(position); - lengthBuffer.position(0); - segment.put(0, lengthBuffer, position); - return Optional.of(segment); - } - - // for the case of full length, partial data in buffer - if (recordLength > THRESHOLD_FOR_SPILLING) { - throw new UnsupportedOperationException("Unaligned checkpoint currently do not support spilled " + - "records."); - } else if (recordLength != -1) { - int leftOverSize = leftOverLimit - leftOverStart; - int unconsumedSize = Integer.BYTES + accumulatedRecordBytes + leftOverSize; - DataOutputSerializer serializer = new DataOutputSerializer(unconsumedSize); - serializer.writeInt(recordLength); - serializer.write(buffer, 0, accumulatedRecordBytes); - if (leftOverData != null) { - serializer.write(leftOverData, leftOverStart, leftOverSize); - } - MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(unconsumedSize); - segment.put(0, serializer.getSharedBuffer(), 0, segment.size()); - return Optional.of(segment); - } - - // for the case of no remaining partial length or data - return Optional.empty(); - } - - private void moveRemainderToNonSpanningDeserializer(NonSpanningWrapper deserializer) { - deserializer.clear(); - - if (leftOverData != null) { - deserializer.initializeFromMemorySegment(leftOverData, leftOverStart, leftOverLimit); - } - } - - private boolean hasFullRecord() { - return this.recordLength >= 0 && this.accumulatedRecordBytes >= this.recordLength; - } - - private int getNumGatheredBytes() { - return this.accumulatedRecordBytes + (this.recordLength >= 0 ? 4 : lengthBuffer.position()); - } - - public void clear() { - this.buffer = initialBuffer; - this.serializationReadBuffer.releaseArrays(); - - this.recordLength = -1; - this.lengthBuffer.clear(); - this.leftOverData = null; - this.leftOverStart = 0; - this.leftOverLimit = 0; - this.accumulatedRecordBytes = 0; - - if (spillingChannel != null) { - try { - spillingChannel.close(); - } - catch (Throwable t) { - // ignore - } - spillingChannel = null; - } - if (spillFileReader != null) { - try { - spillFileReader.close(); - } - catch (Throwable t) { - // ignore - } - spillFileReader = null; - } - if (spillFile != null) { - spillFile.delete(); - spillFile = null; - } - } - - public DataInputView getInputView() { - if (spillFileReader == null) { - return serializationReadBuffer; - } - else { - return spillFileReader; - } - } - - private void ensureBufferCapacity(int minLength) { - if (buffer.length < minLength) { - byte[] newBuffer = new byte[Math.max(minLength, buffer.length * 2)]; - System.arraycopy(buffer, 0, newBuffer, 0, accumulatedRecordBytes); - buffer = newBuffer; - } - } - - @SuppressWarnings("resource") - private FileChannel createSpillingChannel() throws IOException { - if (spillFile != null) { - throw new IllegalStateException("Spilling file already exists."); - } - - // try to find a unique file name for the spilling channel - int maxAttempts = 10; - for (int attempt = 0; attempt < maxAttempts; attempt++) { - String directory = tempDirs[rnd.nextInt(tempDirs.length)]; - spillFile = new File(directory, randomString(rnd) + ".inputchannel"); - if (spillFile.createNewFile()) { - return new RandomAccessFile(spillFile, "rw").getChannel(); - } - } - - throw new IOException( - "Could not find a unique file channel name in '" + Arrays.toString(tempDirs) + - "' for spilling large records during deserialization."); - } - - private static String randomString(Random random) { - final byte[] bytes = new byte[20]; - random.nextBytes(bytes); - return StringUtils.byteToHexString(bytes); - } - } } From c6bdeb4b5b4c608510c98a91e19f5facdad28ed0 Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Tue, 12 May 2020 11:23:39 +0200 Subject: [PATCH 052/773] [FLINK-17547][task][hotfix] Fix compiler warnings in NonSpanningWrapper --- .../api/serialization/NonSpanningWrapper.java | 53 +++++++++---------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java index bab50fafd94aa..6d9602fbe8be1 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java @@ -67,12 +67,12 @@ Optional getUnconsumedSegment() { // ------------------------------------------------------------------------------------------------------------- @Override - public final void readFully(byte[] b) throws IOException { + public final void readFully(byte[] b) { readFully(b, 0, b.length); } @Override - public final void readFully(byte[] b, int off, int len) throws IOException { + public final void readFully(byte[] b, int off, int len) { if (off < 0 || len < 0 || off + len > b.length) { throw new IndexOutOfBoundsException(); } @@ -82,78 +82,75 @@ public final void readFully(byte[] b, int off, int len) throws IOException { } @Override - public final boolean readBoolean() throws IOException { + public final boolean readBoolean() { return readByte() == 1; } @Override - public final byte readByte() throws IOException { + public final byte readByte() { return this.segment.get(this.position++); } @Override - public final int readUnsignedByte() throws IOException { + public final int readUnsignedByte() { return readByte() & 0xff; } @Override - public final short readShort() throws IOException { + public final short readShort() { final short v = this.segment.getShortBigEndian(this.position); this.position += 2; return v; } @Override - public final int readUnsignedShort() throws IOException { + public final int readUnsignedShort() { final int v = this.segment.getShortBigEndian(this.position) & 0xffff; this.position += 2; return v; } @Override - public final char readChar() throws IOException { + public final char readChar() { final char v = this.segment.getCharBigEndian(this.position); this.position += 2; return v; } @Override - public final int readInt() throws IOException { + public final int readInt() { final int v = this.segment.getIntBigEndian(this.position); this.position += 4; return v; } @Override - public final long readLong() throws IOException { + public final long readLong() { final long v = this.segment.getLongBigEndian(this.position); this.position += 8; return v; } @Override - public final float readFloat() throws IOException { + public final float readFloat() { return Float.intBitsToFloat(readInt()); } @Override - public final double readDouble() throws IOException { + public final double readDouble() { return Double.longBitsToDouble(readLong()); } @Override - public final String readLine() throws IOException { + public final String readLine() { final StringBuilder bld = new StringBuilder(32); - try { - int b; - while ((b = readUnsignedByte()) != '\n') { - if (b != '\r') { - bld.append((char) b); - } + int b; + while ((b = readUnsignedByte()) != '\n') { + if (b != '\r') { + bld.append((char) b); } } - catch (EOFException ignored) {} if (bld.length() == 0) { return null; @@ -168,7 +165,7 @@ public final String readLine() throws IOException { } @Override - public final String readUTF() throws IOException { + public final String readUTF() throws UTFDataFormatException { final int utflen = readUnsignedShort(); final byte[] bytearr; @@ -222,7 +219,7 @@ public final String readUTF() throws IOException { if (count > utflen) { throw new UTFDataFormatException("malformed input: partial character at end"); } - char2 = (int) bytearr[count - 1]; + char2 = bytearr[count - 1]; if ((char2 & 0xC0) != 0x80) { throw new UTFDataFormatException("malformed input around byte " + count); } @@ -233,8 +230,8 @@ public final String readUTF() throws IOException { if (count > utflen) { throw new UTFDataFormatException("malformed input: partial character at end"); } - char2 = (int) bytearr[count - 2]; - char3 = (int) bytearr[count - 1]; + char2 = bytearr[count - 2]; + char3 = bytearr[count - 1]; if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { throw new UTFDataFormatException("malformed input around byte " + (count - 1)); } @@ -249,7 +246,7 @@ public final String readUTF() throws IOException { } @Override - public final int skipBytes(int n) throws IOException { + public final int skipBytes(int n) { if (n < 0) { throw new IllegalArgumentException(); } @@ -260,7 +257,7 @@ public final int skipBytes(int n) throws IOException { } @Override - public void skipBytesToRead(int numBytes) throws IOException { + public void skipBytesToRead(int numBytes) throws EOFException { int skippedBytes = skipBytes(numBytes); if (skippedBytes < numBytes){ @@ -269,7 +266,7 @@ public void skipBytesToRead(int numBytes) throws IOException { } @Override - public int read(byte[] b, int off, int len) throws IOException { + public int read(byte[] b, int off, int len) { if (b == null){ throw new NullPointerException("Byte array b cannot be null."); } @@ -290,7 +287,7 @@ public int read(byte[] b, int off, int len) throws IOException { } @Override - public int read(byte[] b) throws IOException { + public int read(byte[] b) { return read(b, 0, b.length); } } From 9ebdaf40973e1b7ff6aaadd19d1d6cda1d3e69d8 Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Tue, 12 May 2020 11:24:01 +0200 Subject: [PATCH 053/773] [FLINK-17547][task][hotfix] Extract methods from RecordsDeserializer --- .../java/org/apache/flink/util/IOUtils.java | 9 + .../api/serialization/NonSpanningWrapper.java | 81 ++++- .../api/serialization/SpanningWrapper.java | 278 +++++++++--------- ...ingAdaptiveSpanningRecordDeserializer.java | 121 ++++---- 4 files changed, 271 insertions(+), 218 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/util/IOUtils.java b/flink-core/src/main/java/org/apache/flink/util/IOUtils.java index 02b11e66130f4..1f9af1858ea66 100644 --- a/flink-core/src/main/java/org/apache/flink/util/IOUtils.java +++ b/flink-core/src/main/java/org/apache/flink/util/IOUtils.java @@ -26,6 +26,8 @@ import java.io.PrintStream; import java.net.Socket; +import static java.util.Arrays.asList; + /** * An utility class for I/O related functionality. */ @@ -241,6 +243,13 @@ public static void closeAll(Iterable closeables) throws } } + /** + * Closes all elements in the iterable with closeQuietly(). + */ + public static void closeAllQuietly(AutoCloseable... closeables) { + closeAllQuietly(asList(closeables)); + } + /** * Closes all elements in the iterable with closeQuietly(). */ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java index 6d9602fbe8be1..5de546776a4a2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java @@ -17,27 +17,43 @@ package org.apache.flink.runtime.io.network.api.serialization; +import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.NextRecordResponse; import java.io.EOFException; import java.io.IOException; import java.io.UTFDataFormatException; +import java.nio.ByteBuffer; import java.util.Optional; +import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; +import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER; +import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.PARTIAL_RECORD; +import static org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.LENGTH_BYTES; + final class NonSpanningWrapper implements DataInputView { - MemorySegment segment; + private static final String BROKEN_SERIALIZATION_ERROR_MESSAGE = + "Serializer consumed more bytes than the record had. " + + "This indicates broken serialization. If you are using custom serialization types " + + "(Value or Writable), check their serialization methods. If you are using a " + + "Kryo-serialized type, check the corresponding Kryo serializer."; + + private MemorySegment segment; private int limit; - int position; + private int position; private byte[] utfByteBuffer; // reusable byte buffer for utf-8 decoding private char[] utfCharBuffer; // reusable char buffer for utf-8 decoding - int remaining() { + private final NextRecordResponse reusedNextRecordResponse = new NextRecordResponse(null, 0); // performance impact of immutable objects not benchmarked + + private int remaining() { return this.limit - this.position; } @@ -47,14 +63,14 @@ void clear() { this.position = 0; } - void initializeFromMemorySegment(MemorySegment seg, int position, int leftOverLimit) { + void initializeFromMemorySegment(MemorySegment seg, int position, int limit) { this.segment = seg; this.position = position; - this.limit = leftOverLimit; + this.limit = limit; } Optional getUnconsumedSegment() { - if (remaining() == 0) { + if (!hasRemaining()) { return Optional.empty(); } MemorySegment target = MemorySegmentFactory.allocateUnpooledSegment(remaining()); @@ -62,6 +78,10 @@ Optional getUnconsumedSegment() { return Optional.of(target); } + boolean hasRemaining() { + return remaining() > 0; + } + // ------------------------------------------------------------------------------------------------------------- // DataInput specific methods // ------------------------------------------------------------------------------------------------------------- @@ -290,4 +310,53 @@ public int read(byte[] b, int off, int len) { public int read(byte[] b) { return read(b, 0, b.length); } + + ByteBuffer wrapIntoByteBuffer() { + return segment.wrap(position, remaining()); + } + + int copyContentTo(byte[] dst) { + final int numBytesChunk = remaining(); + segment.get(position, dst, 0, numBytesChunk); + return numBytesChunk; + } + + /** + * Copies the data and transfers the "ownership" (i.e. clears current wrapper). + */ + void transferTo(ByteBuffer dst) { + segment.get(position, dst, remaining()); + clear(); + } + + NextRecordResponse getNextRecord(IOReadableWritable target) throws IOException { + int recordLen = readInt(); + if (canReadRecord(recordLen)) { + return readInto(target); + } else { + return reusedNextRecordResponse.updated(PARTIAL_RECORD, recordLen); + } + } + + private NextRecordResponse readInto(IOReadableWritable target) throws IOException { + try { + target.read(this); + } catch (IndexOutOfBoundsException e) { + throw new IOException(BROKEN_SERIALIZATION_ERROR_MESSAGE, e); + } + int remaining = remaining(); + if (remaining < 0) { + throw new IOException(BROKEN_SERIALIZATION_ERROR_MESSAGE, new IndexOutOfBoundsException("Remaining = " + remaining)); + } + return reusedNextRecordResponse.updated(remaining == 0 ? LAST_RECORD_FROM_BUFFER : INTERMEDIATE_RECORD_FROM_BUFFER, remaining); + } + + boolean hasCompleteLength() { + return remaining() >= LENGTH_BYTES; + } + + private boolean canReadRecord(int recordLength) { + return recordLength <= remaining(); + } + } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java index e59363f7c2865..430f0db06496c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java @@ -23,7 +23,6 @@ import org.apache.flink.core.memory.DataOutputSerializer; import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemorySegmentFactory; -import org.apache.flink.util.FileUtils; import org.apache.flink.util.StringUtils; import java.io.BufferedInputStream; @@ -38,9 +37,16 @@ import java.util.Optional; import java.util.Random; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.LENGTH_BYTES; +import static org.apache.flink.util.FileUtils.writeCompletely; +import static org.apache.flink.util.IOUtils.closeAllQuietly; + final class SpanningWrapper { private static final int THRESHOLD_FOR_SPILLING = 5 * 1024 * 1024; // 5 MiBytes + private static final int FILE_BUFFER_SIZE = 2 * 1024 * 1024; private final byte[] initialBuffer = new byte[1024]; @@ -50,7 +56,7 @@ final class SpanningWrapper { private final DataInputDeserializer serializationReadBuffer; - private final ByteBuffer lengthBuffer; + final ByteBuffer lengthBuffer; private FileChannel spillingChannel; @@ -70,10 +76,10 @@ final class SpanningWrapper { private DataInputViewStreamWrapper spillFileReader; - public SpanningWrapper(String[] tempDirs) { + SpanningWrapper(String[] tempDirs) { this.tempDirs = tempDirs; - this.lengthBuffer = ByteBuffer.allocate(4); + this.lengthBuffer = ByteBuffer.allocate(LENGTH_BYTES); this.lengthBuffer.order(ByteOrder.BIG_ENDIAN); this.recordLength = -1; @@ -82,187 +88,161 @@ public SpanningWrapper(String[] tempDirs) { this.buffer = initialBuffer; } - void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException { - // set the length and copy what is available to the buffer - this.recordLength = nextRecordLength; - - final int numBytesChunk = partial.remaining(); - - if (nextRecordLength > THRESHOLD_FOR_SPILLING) { - // create a spilling channel and put the data there - this.spillingChannel = createSpillingChannel(); - - ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk); - FileUtils.writeCompletely(this.spillingChannel, toWrite); - } - else { - // collect in memory - ensureBufferCapacity(nextRecordLength); - partial.segment.get(partial.position, buffer, 0, numBytesChunk); - } - - this.accumulatedRecordBytes = numBytesChunk; + /** + * Copies the data and transfers the "ownership" (i.e. clears the passed wrapper). + */ + void transferFrom(NonSpanningWrapper partial, int nextRecordLength) throws IOException { + updateLength(nextRecordLength); + accumulatedRecordBytes = isAboveSpillingThreshold() ? spill(partial) : partial.copyContentTo(buffer); + partial.clear(); } - void initializeWithPartialLength(NonSpanningWrapper partial) throws IOException { - // copy what we have to the length buffer - partial.segment.get(partial.position, this.lengthBuffer, partial.remaining()); + private boolean isAboveSpillingThreshold() { + return recordLength > THRESHOLD_FOR_SPILLING; } void addNextChunkFromMemorySegment(MemorySegment segment, int offset, int numBytes) throws IOException { - int segmentPosition = offset; - int segmentRemaining = numBytes; - // check where to go. if we have a partial length, we need to complete it first - if (this.lengthBuffer.position() > 0) { - int toPut = Math.min(this.lengthBuffer.remaining(), segmentRemaining); - segment.get(segmentPosition, this.lengthBuffer, toPut); - // did we complete the length? - if (this.lengthBuffer.hasRemaining()) { - return; - } else { - this.recordLength = this.lengthBuffer.getInt(0); - - this.lengthBuffer.clear(); - segmentPosition += toPut; - segmentRemaining -= toPut; - if (this.recordLength > THRESHOLD_FOR_SPILLING) { - this.spillingChannel = createSpillingChannel(); - } else { - ensureBufferCapacity(this.recordLength); - } - } + int limit = offset + numBytes; + int numBytesRead = isReadingLength() ? readLength(segment, offset, numBytes) : 0; + offset += numBytesRead; + numBytes -= numBytesRead; + if (numBytes == 0) { + return; } - // copy as much as we need or can for this next spanning record - int needed = this.recordLength - this.accumulatedRecordBytes; - int toCopy = Math.min(needed, segmentRemaining); + int toCopy = min(recordLength - accumulatedRecordBytes, numBytes); + if (toCopy > 0) { + copyFromSegment(segment, offset, toCopy); + } + if (numBytes > toCopy) { + leftOverData = segment; + leftOverStart = offset + toCopy; + leftOverLimit = limit; + } + } - if (spillingChannel != null) { - // spill to file - ByteBuffer toWrite = segment.wrap(segmentPosition, toCopy); - FileUtils.writeCompletely(this.spillingChannel, toWrite); + private void copyFromSegment(MemorySegment segment, int offset, int length) throws IOException { + if (spillingChannel == null) { + copyIntoBuffer(segment, offset, length); } else { - segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy); + copyIntoFile(segment, offset, length); } + } - this.accumulatedRecordBytes += toCopy; - - if (toCopy < segmentRemaining) { - // there is more data in the segment - this.leftOverData = segment; - this.leftOverStart = segmentPosition + toCopy; - this.leftOverLimit = numBytes + offset; + private void copyIntoFile(MemorySegment segment, int offset, int length) throws IOException { + writeCompletely(spillingChannel, segment.wrap(offset, length)); + accumulatedRecordBytes += length; + if (hasFullRecord()) { + spillingChannel.close(); + spillFileReader = new DataInputViewStreamWrapper(new BufferedInputStream(new FileInputStream(spillFile), FILE_BUFFER_SIZE)); } + } - if (accumulatedRecordBytes == recordLength) { - // we have the full record - if (spillingChannel == null) { - this.serializationReadBuffer.setBuffer(buffer, 0, recordLength); - } - else { - spillingChannel.close(); + private void copyIntoBuffer(MemorySegment segment, int offset, int length) { + segment.get(offset, buffer, accumulatedRecordBytes, length); + accumulatedRecordBytes += length; + if (hasFullRecord()) { + serializationReadBuffer.setBuffer(buffer, 0, recordLength); + } + } - BufferedInputStream inStream = new BufferedInputStream(new FileInputStream(spillFile), 2 * 1024 * 1024); - this.spillFileReader = new DataInputViewStreamWrapper(inStream); - } + private int readLength(MemorySegment segment, int segmentPosition, int segmentRemaining) throws IOException { + int bytesToRead = min(lengthBuffer.remaining(), segmentRemaining); + segment.get(segmentPosition, lengthBuffer, bytesToRead); + if (!lengthBuffer.hasRemaining()) { + updateLength(lengthBuffer.getInt(0)); } + return bytesToRead; } - Optional getUnconsumedSegment() throws IOException { - // for the case of only partial length, no data - final int position = lengthBuffer.position(); - if (position > 0) { - MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(position); - lengthBuffer.position(0); - segment.put(0, lengthBuffer, position); - return Optional.of(segment); + private void updateLength(int length) throws IOException { + lengthBuffer.clear(); + recordLength = length; + if (isAboveSpillingThreshold()) { + spillingChannel = createSpillingChannel(); + } else { + ensureBufferCapacity(length); } + } - // for the case of full length, partial data in buffer - if (recordLength > THRESHOLD_FOR_SPILLING) { - throw new UnsupportedOperationException("Unaligned checkpoint currently do not support spilled " + - "records."); - } else if (recordLength != -1) { - int leftOverSize = leftOverLimit - leftOverStart; - int unconsumedSize = Integer.BYTES + accumulatedRecordBytes + leftOverSize; - DataOutputSerializer serializer = new DataOutputSerializer(unconsumedSize); - serializer.writeInt(recordLength); - serializer.write(buffer, 0, accumulatedRecordBytes); - if (leftOverData != null) { - serializer.write(leftOverData, leftOverStart, leftOverSize); - } - MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(unconsumedSize); - segment.put(0, serializer.getSharedBuffer(), 0, segment.size()); - return Optional.of(segment); + Optional getUnconsumedSegment() throws IOException { + if (isReadingLength()) { + return Optional.of(copyLengthBuffer()); + } else if (isAboveSpillingThreshold()) { + throw new UnsupportedOperationException("Unaligned checkpoint currently do not support spilled records."); + } else if (recordLength == -1) { + return Optional.empty(); // no remaining partial length or data + } else { + return Optional.of(copyDataBuffer()); } + } - // for the case of no remaining partial length or data - return Optional.empty(); + private MemorySegment copyLengthBuffer() { + int position = lengthBuffer.position(); + MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(position); + lengthBuffer.position(0); + segment.put(0, lengthBuffer, position); + return segment; } - void moveRemainderToNonSpanningDeserializer(NonSpanningWrapper deserializer) { - deserializer.clear(); + private MemorySegment copyDataBuffer() throws IOException { + int leftOverSize = leftOverLimit - leftOverStart; + int unconsumedSize = LENGTH_BYTES + accumulatedRecordBytes + leftOverSize; + DataOutputSerializer serializer = new DataOutputSerializer(unconsumedSize); + serializer.writeInt(recordLength); + serializer.write(buffer, 0, accumulatedRecordBytes); + if (leftOverData != null) { + serializer.write(leftOverData, leftOverStart, leftOverSize); + } + MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(unconsumedSize); + segment.put(0, serializer.getSharedBuffer(), 0, segment.size()); + return segment; + } + /** + * Copies the leftover data and transfers the "ownership" (i.e. clears this wrapper). + */ + void transferLeftOverTo(NonSpanningWrapper nonSpanningWrapper) { + nonSpanningWrapper.clear(); if (leftOverData != null) { - deserializer.initializeFromMemorySegment(leftOverData, leftOverStart, leftOverLimit); + nonSpanningWrapper.initializeFromMemorySegment(leftOverData, leftOverStart, leftOverLimit); } + clear(); } boolean hasFullRecord() { - return this.recordLength >= 0 && this.accumulatedRecordBytes >= this.recordLength; + return recordLength >= 0 && accumulatedRecordBytes >= recordLength; } int getNumGatheredBytes() { - return this.accumulatedRecordBytes + (this.recordLength >= 0 ? 4 : lengthBuffer.position()); + return accumulatedRecordBytes + (recordLength >= 0 ? LENGTH_BYTES : lengthBuffer.position()); } + @SuppressWarnings("ResultOfMethodCallIgnored") public void clear() { - this.buffer = initialBuffer; - this.serializationReadBuffer.releaseArrays(); - - this.recordLength = -1; - this.lengthBuffer.clear(); - this.leftOverData = null; - this.leftOverStart = 0; - this.leftOverLimit = 0; - this.accumulatedRecordBytes = 0; - - if (spillingChannel != null) { - try { - spillingChannel.close(); - } - catch (Throwable t) { - // ignore - } - spillingChannel = null; - } - if (spillFileReader != null) { - try { - spillFileReader.close(); - } - catch (Throwable t) { - // ignore - } - spillFileReader = null; - } - if (spillFile != null) { - spillFile.delete(); - spillFile = null; - } + buffer = initialBuffer; + serializationReadBuffer.releaseArrays(); + + recordLength = -1; + lengthBuffer.clear(); + leftOverData = null; + leftOverStart = 0; + leftOverLimit = 0; + accumulatedRecordBytes = 0; + + closeAllQuietly(spillingChannel, spillFileReader, () -> spillFile.delete()); + spillingChannel = null; + spillFileReader = null; + spillFile = null; } public DataInputView getInputView() { - if (spillFileReader == null) { - return serializationReadBuffer; - } - else { - return spillFileReader; - } + return spillFileReader == null ? serializationReadBuffer : spillFileReader; } private void ensureBufferCapacity(int minLength) { if (buffer.length < minLength) { - byte[] newBuffer = new byte[Math.max(minLength, buffer.length * 2)]; + byte[] newBuffer = new byte[max(minLength, buffer.length * 2)]; System.arraycopy(buffer, 0, newBuffer, 0, accumulatedRecordBytes); buffer = newBuffer; } @@ -294,4 +274,16 @@ private static String randomString(Random random) { random.nextBytes(bytes); return StringUtils.byteToHexString(bytes); } + + private int spill(NonSpanningWrapper partial) throws IOException { + ByteBuffer buffer = partial.wrapIntoByteBuffer(); + int length = buffer.remaining(); + writeCompletely(spillingChannel, buffer); + return length; + } + + private boolean isReadingLength() { + return lengthBuffer.position() > 0; + } + } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java index f20fbc92c73b8..75e6b0bfc5587 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java @@ -24,19 +24,22 @@ import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; +import javax.annotation.concurrent.NotThreadSafe; + import java.io.IOException; import java.util.Optional; +import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; +import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER; +import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.PARTIAL_RECORD; +import static org.apache.flink.runtime.io.network.buffer.Buffer.DataType.DATA_BUFFER; + /** * @param The type of the record to be deserialized. */ public class SpillingAdaptiveSpanningRecordDeserializer implements RecordDeserializer { - private static final String BROKEN_SERIALIZATION_ERROR_MESSAGE = - "Serializer consumed more bytes than the record had. " + - "This indicates broken serialization. If you are using custom serialization types " + - "(Value or Writable), check their serialization methods. If you are using a " + - "Kryo-serialized type, check the corresponding Kryo serializer."; + static final int LENGTH_BYTES = Integer.BYTES; private final NonSpanningWrapper nonSpanningWrapper; @@ -58,11 +61,10 @@ public void setNextBuffer(Buffer buffer) throws IOException { int numBytes = buffer.getSize(); // check if some spanning record deserialization is pending - if (this.spanningWrapper.getNumGatheredBytes() > 0) { - this.spanningWrapper.addNextChunkFromMemorySegment(segment, offset, numBytes); - } - else { - this.nonSpanningWrapper.initializeFromMemorySegment(segment, offset, numBytes + offset); + if (spanningWrapper.getNumGatheredBytes() > 0) { + spanningWrapper.addNextChunkFromMemorySegment(segment, offset, numBytes); + } else { + nonSpanningWrapper.initializeFromMemorySegment(segment, offset, numBytes + offset); } } @@ -75,14 +77,13 @@ public Buffer getCurrentBuffer () { @Override public Optional getUnconsumedBuffer() throws IOException { - Optional target; - if (nonSpanningWrapper.remaining() > 0) { - target = nonSpanningWrapper.getUnconsumedSegment(); + final Optional unconsumedSegment; + if (nonSpanningWrapper.hasRemaining()) { + unconsumedSegment = nonSpanningWrapper.getUnconsumedSegment(); } else { - target = spanningWrapper.getUnconsumedSegment(); + unconsumedSegment = spanningWrapper.getUnconsumedSegment(); } - return target.map(memorySegment -> new NetworkBuffer( - memorySegment, FreeingBufferRecycler.INSTANCE, Buffer.DataType.DATA_BUFFER, memorySegment.size())); + return unconsumedSegment.map(segment -> new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, DATA_BUFFER, segment.size())); } @Override @@ -91,65 +92,31 @@ public DeserializationResult getNextRecord(T target) throws IOException { // this should be the majority of the cases for small records // for large records, this portion of the work is very small in comparison anyways - int nonSpanningRemaining = this.nonSpanningWrapper.remaining(); - - // check if we can get a full length; - if (nonSpanningRemaining >= 4) { - int len = this.nonSpanningWrapper.readInt(); - - if (len <= nonSpanningRemaining - 4) { - // we can get a full record from here - try { - target.read(this.nonSpanningWrapper); - - int remaining = this.nonSpanningWrapper.remaining(); - if (remaining > 0) { - return DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; - } - else if (remaining == 0) { - return DeserializationResult.LAST_RECORD_FROM_BUFFER; - } - else { - throw new IndexOutOfBoundsException("Remaining = " + remaining); - } - } - catch (IndexOutOfBoundsException e) { - throw new IOException(BROKEN_SERIALIZATION_ERROR_MESSAGE, e); - } - } - else { - // we got the length, but we need the rest from the spanning deserializer - // and need to wait for more buffers - this.spanningWrapper.initializeWithPartialRecord(this.nonSpanningWrapper, len); - this.nonSpanningWrapper.clear(); - return DeserializationResult.PARTIAL_RECORD; - } - } else if (nonSpanningRemaining > 0) { - // we have an incomplete length - // add our part of the length to the length buffer - this.spanningWrapper.initializeWithPartialLength(this.nonSpanningWrapper); - this.nonSpanningWrapper.clear(); - return DeserializationResult.PARTIAL_RECORD; - } + if (nonSpanningWrapper.hasCompleteLength()) { + return readNonSpanningRecord(target); - // spanning record case - if (this.spanningWrapper.hasFullRecord()) { - // get the full record - target.read(this.spanningWrapper.getInputView()); + } else if (nonSpanningWrapper.hasRemaining()) { + nonSpanningWrapper.transferTo(spanningWrapper.lengthBuffer); + return PARTIAL_RECORD; - // move the remainder to the non-spanning wrapper - // this does not copy it, only sets the memory segment - this.spanningWrapper.moveRemainderToNonSpanningDeserializer(this.nonSpanningWrapper); - this.spanningWrapper.clear(); + } else if (spanningWrapper.hasFullRecord()) { + target.read(spanningWrapper.getInputView()); + spanningWrapper.transferLeftOverTo(nonSpanningWrapper); + return nonSpanningWrapper.hasRemaining() ? INTERMEDIATE_RECORD_FROM_BUFFER : LAST_RECORD_FROM_BUFFER; - return (this.nonSpanningWrapper.remaining() == 0) ? - DeserializationResult.LAST_RECORD_FROM_BUFFER : - DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; } else { - return DeserializationResult.PARTIAL_RECORD; + return PARTIAL_RECORD; } } + private DeserializationResult readNonSpanningRecord(T target) throws IOException { + NextRecordResponse response = nonSpanningWrapper.getNextRecord(target); + if (response.result == PARTIAL_RECORD) { + spanningWrapper.transferFrom(nonSpanningWrapper, response.bytesLeft); + } + return response.result; + } + @Override public void clear() { this.nonSpanningWrapper.clear(); @@ -158,7 +125,23 @@ public void clear() { @Override public boolean hasUnfinishedData() { - return this.nonSpanningWrapper.remaining() > 0 || this.spanningWrapper.getNumGatheredBytes() > 0; + return this.nonSpanningWrapper.hasRemaining() || this.spanningWrapper.getNumGatheredBytes() > 0; } + @NotThreadSafe + static class NextRecordResponse { + DeserializationResult result; + int bytesLeft; + + NextRecordResponse(DeserializationResult result, int bytesLeft) { + this.result = result; + this.bytesLeft = bytesLeft; + } + + public NextRecordResponse updated(DeserializationResult result, int bytesLeft) { + this.result = result; + this.bytesLeft = bytesLeft; + return this; + } + } } From 1c9bf0368e9a233a2a013436628790c2c2b60bcb Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Mon, 18 May 2020 20:29:05 +0200 Subject: [PATCH 054/773] [FLINK-17547][task] Use iterator for unconsumed buffers. Motivation: support spilled records Changes: 1. change SpillingAdaptiveSpanningRecordDeserializer.getUnconsumedBuffer signature 2. adapt channel state persistence to new types No changes in existing logic. --- .../apache/flink/util/CloseableIterator.java | 77 ++++++++++++++++++- .../java/org/apache/flink/util/IOUtils.java | 7 ++ .../channel/ChannelStateWriteRequest.java | 33 ++++++-- ...hannelStateWriteRequestDispatcherImpl.java | 6 +- .../ChannelStateWriteRequestExecutorImpl.java | 20 +++-- .../channel/ChannelStateWriter.java | 6 +- .../channel/ChannelStateWriterImpl.java | 17 ++-- .../api/serialization/NonSpanningWrapper.java | 22 ++++-- .../api/serialization/RecordDeserializer.java | 4 +- .../api/serialization/SpanningWrapper.java | 12 +-- ...ingAdaptiveSpanningRecordDeserializer.java | 15 +--- .../consumer/RemoteInputChannel.java | 3 +- ...hannelStateWriteRequestDispatcherTest.java | 10 ++- ...nnelStateWriteRequestExecutorImplTest.java | 1 - .../channel/ChannelStateWriterImplTest.java | 13 ++-- .../CheckpointInProgressRequestTest.java | 7 +- .../channel/MockChannelStateWriter.java | 11 ++- .../channel/RecordingChannelStateWriter.java | 12 ++- .../SpanningRecordSerializationTest.java | 9 ++- .../consumer/SingleInputGateTest.java | 14 +++- .../state/ChannelPersistenceITCase.java | 3 +- .../io/CheckpointBarrierUnaligner.java | 3 +- .../runtime/io/StreamTaskNetworkInput.java | 11 ++- 23 files changed, 235 insertions(+), 81 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java b/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java index 09ea0461a2292..cc51324df93d7 100644 --- a/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java +++ b/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java @@ -20,10 +20,15 @@ import javax.annotation.Nonnull; +import java.util.ArrayDeque; import java.util.Collections; +import java.util.Deque; import java.util.Iterator; +import java.util.List; import java.util.function.Consumer; +import static java.util.Arrays.asList; + /** * This interface represents an {@link Iterator} that is also {@link AutoCloseable}. A typical use-case for this * interface are iterators that are based on native-resources such as files, network, or database connections. Clients @@ -37,7 +42,42 @@ public interface CloseableIterator extends Iterator, AutoCloseable { @Nonnull static CloseableIterator adapterForIterator(@Nonnull Iterator iterator) { - return new IteratorAdapter<>(iterator); + return adapterForIterator(iterator, () -> {}); + } + + static CloseableIterator adapterForIterator(@Nonnull Iterator iterator, AutoCloseable close) { + return new IteratorAdapter<>(iterator, close); + } + + static CloseableIterator fromList(List list, Consumer closeNotConsumed) { + return new CloseableIterator(){ + private final Deque stack = new ArrayDeque<>(list); + + @Override + public boolean hasNext() { + return !stack.isEmpty(); + } + + @Override + public T next() { + return stack.poll(); + } + + @Override + public void close() throws Exception { + Exception exception = null; + for (T el : stack) { + try { + closeNotConsumed.accept(el); + } catch (Exception e) { + exception = ExceptionUtils.firstOrSuppressed(e, exception); + } + } + if (exception != null) { + throw exception; + } + } + }; } @SuppressWarnings("unchecked") @@ -45,6 +85,34 @@ static CloseableIterator empty() { return (CloseableIterator) EMPTY_INSTANCE; } + static CloseableIterator ofElements(Consumer closeNotConsumed, T... elements) { + return fromList(asList(elements), closeNotConsumed); + } + + static CloseableIterator ofElement(E element, Consumer closeIfNotConsumed) { + return new CloseableIterator(){ + private boolean hasNext = true; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public E next() { + hasNext = false; + return element; + } + + @Override + public void close() { + if (hasNext) { + closeIfNotConsumed.accept(element); + } + } + }; + } + /** * Adapter from {@link Iterator} to {@link CloseableIterator}. Does nothing on {@link #close()}. * @@ -54,9 +122,11 @@ final class IteratorAdapter implements CloseableIterator { @Nonnull private final Iterator delegate; + private final AutoCloseable close; - IteratorAdapter(@Nonnull Iterator delegate) { + IteratorAdapter(@Nonnull Iterator delegate, AutoCloseable close) { this.delegate = delegate; + this.close = close; } @Override @@ -80,7 +150,8 @@ public void forEachRemaining(Consumer action) { } @Override - public void close() { + public void close() throws Exception { + close.close(); } } } diff --git a/flink-core/src/main/java/org/apache/flink/util/IOUtils.java b/flink-core/src/main/java/org/apache/flink/util/IOUtils.java index 1f9af1858ea66..0b8f210a8187f 100644 --- a/flink-core/src/main/java/org/apache/flink/util/IOUtils.java +++ b/flink-core/src/main/java/org/apache/flink/util/IOUtils.java @@ -215,6 +215,13 @@ public static void closeSocket(final Socket sock) { } } + /** + * @see #closeAll(Iterable) + */ + public static void closeAll(AutoCloseable... closeables) throws Exception { + closeAll(asList(closeables)); + } + /** * Closes all {@link AutoCloseable} objects in the parameter, suppressing exceptions. Exception will be emitted * after calling close() on every object. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequest.java index bd6c7ba4567f7..084869867fca3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequest.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequest.java @@ -20,23 +20,24 @@ import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter.ChannelStateWriteResult; import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.state.CheckpointStorageLocationReference; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.ThrowingConsumer; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import static org.apache.flink.runtime.checkpoint.channel.CheckpointInProgressRequestState.CANCELLED; import static org.apache.flink.runtime.checkpoint.channel.CheckpointInProgressRequestState.COMPLETED; import static org.apache.flink.runtime.checkpoint.channel.CheckpointInProgressRequestState.EXECUTING; import static org.apache.flink.runtime.checkpoint.channel.CheckpointInProgressRequestState.FAILED; import static org.apache.flink.runtime.checkpoint.channel.CheckpointInProgressRequestState.NEW; +import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; interface ChannelStateWriteRequest { long getCheckpointId(); - void cancel(Throwable cause); + void cancel(Throwable cause) throws Exception; static CheckpointInProgressRequest completeInput(long checkpointId) { return new CheckpointInProgressRequest("completeInput", checkpointId, ChannelStateCheckpointWriter::completeInput, false); @@ -46,8 +47,24 @@ static CheckpointInProgressRequest completeOutput(long checkpointId) { return new CheckpointInProgressRequest("completeOutput", checkpointId, ChannelStateCheckpointWriter::completeOutput, false); } - static ChannelStateWriteRequest write(long checkpointId, InputChannelInfo info, Buffer... flinkBuffers) { - return new CheckpointInProgressRequest("writeInput", checkpointId, writer -> writer.writeInput(info, flinkBuffers), recycle(flinkBuffers), false); + static ChannelStateWriteRequest write(long checkpointId, InputChannelInfo info, CloseableIterator iterator) { + return new CheckpointInProgressRequest( + "writeInput", + checkpointId, + writer -> { + while (iterator.hasNext()) { + Buffer buffer = iterator.next(); + try { + checkArgument(buffer.isBuffer()); + } catch (Exception e) { + buffer.recycleBuffer(); + throw e; + } + writer.writeInput(info, buffer); + } + }, + throwable -> iterator.close(), + false); } static ChannelStateWriteRequest write(long checkpointId, ResultSubpartitionInfo info, Buffer... flinkBuffers) { @@ -62,7 +79,7 @@ static ChannelStateWriteRequest abort(long checkpointId, Throwable cause) { return new CheckpointInProgressRequest("abort", checkpointId, writer -> writer.fail(cause), true); } - static Consumer recycle(Buffer[] flinkBuffers) { + static ThrowingConsumer recycle(Buffer[] flinkBuffers) { return unused -> { for (Buffer b : flinkBuffers) { b.recycleBuffer(); @@ -112,7 +129,7 @@ enum CheckpointInProgressRequestState { final class CheckpointInProgressRequest implements ChannelStateWriteRequest { private final ThrowingConsumer action; - private final Consumer discardAction; + private final ThrowingConsumer discardAction; private final long checkpointId; private final String name; private final boolean ignoreMissingWriter; @@ -123,7 +140,7 @@ final class CheckpointInProgressRequest implements ChannelStateWriteRequest { }, ignoreMissingWriter); } - CheckpointInProgressRequest(String name, long checkpointId, ThrowingConsumer action, Consumer discardAction, boolean ignoreMissingWriter) { + CheckpointInProgressRequest(String name, long checkpointId, ThrowingConsumer action, ThrowingConsumer discardAction, boolean ignoreMissingWriter) { this.checkpointId = checkpointId; this.action = checkNotNull(action); this.discardAction = checkNotNull(discardAction); @@ -137,7 +154,7 @@ public long getCheckpointId() { } @Override - public void cancel(Throwable cause) { + public void cancel(Throwable cause) throws Exception { if (state.compareAndSet(NEW, CANCELLED) || state.compareAndSet(FAILED, CANCELLED)) { discardAction.accept(cause); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherImpl.java index 843663e0c0c3d..0a15b91147dde 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherImpl.java @@ -51,7 +51,11 @@ public void dispatch(ChannelStateWriteRequest request) throws Exception { try { dispatchInternal(request); } catch (Exception e) { - request.cancel(e); + try { + request.cancel(e); + } catch (Exception ex) { + e.addSuppressed(ex); + } throw e; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java index cbcc3f78b89e3..e87a21cadb500 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java @@ -32,6 +32,9 @@ import java.util.concurrent.BlockingDeque; import java.util.concurrent.CancellationException; import java.util.concurrent.LinkedBlockingDeque; +import java.util.stream.Collectors; + +import static org.apache.flink.util.IOUtils.closeAll; /** * Executes {@link ChannelStateWriteRequest}s in a separate thread. Any exception occurred during execution causes this @@ -67,8 +70,15 @@ void run() { } catch (Exception ex) { thrown = ex; } finally { - cleanupRequests(); - dispatcher.fail(thrown == null ? new CancellationException() : thrown); + try { + closeAll( + this::cleanupRequests, + () -> dispatcher.fail(thrown == null ? new CancellationException() : thrown) + ); + } catch (Exception e) { + //noinspection NonAtomicOperationOnVolatileField + thrown = ExceptionUtils.firstOrSuppressed(e, thrown); + } } LOG.debug("loop terminated"); } @@ -87,14 +97,12 @@ private void loop() throws Exception { } } - private void cleanupRequests() { + private void cleanupRequests() throws Exception { Throwable cause = thrown == null ? new CancellationException() : thrown; List drained = new ArrayList<>(); deque.drainTo(drained); LOG.info("discarding {} drained requests", drained.size()); - for (ChannelStateWriteRequest request : drained) { - request.cancel(cause); - } + closeAll(drained.stream().map(request -> () -> request.cancel(cause)).collect(Collectors.toList())); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriter.java index e19b1e2a25df3..5dad559741c06 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriter.java @@ -22,6 +22,7 @@ import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.state.InputChannelStateHandle; import org.apache.flink.runtime.state.ResultSubpartitionStateHandle; +import org.apache.flink.util.CloseableIterator; import java.io.Closeable; import java.util.Collection; @@ -99,11 +100,10 @@ boolean isDone() { * It is intended to use for incremental snapshots. * If no data is passed it is ignored. * @param data zero or more data buffers ordered by their sequence numbers - * @throws IllegalArgumentException if one or more passed buffers {@link Buffer#isBuffer() isn't a buffer} * @see org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter#SEQUENCE_NUMBER_RESTORED * @see org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter#SEQUENCE_NUMBER_UNKNOWN */ - void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, Buffer... data) throws IllegalArgumentException; + void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator data); /** * Add in-flight buffers from the {@link org.apache.flink.runtime.io.network.partition.ResultSubpartition ResultSubpartition}. @@ -161,7 +161,7 @@ public void start(long checkpointId, CheckpointOptions checkpointOptions) { } @Override - public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, Buffer... data) { + public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator data) { } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java index 412a9f520ca7e..b6fa58841adc6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java @@ -24,6 +24,7 @@ import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.state.CheckpointStorageWorkerView; import org.apache.flink.runtime.state.CheckpointStreamFactory; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; @@ -103,10 +104,9 @@ public void start(long checkpointId, CheckpointOptions checkpointOptions) { } @Override - public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, Buffer... data) { - LOG.debug("add input data, checkpoint id: {}, channel: {}, startSeqNum: {}, num buffers: {}", - checkpointId, info, startSeqNum, data == null ? 0 : data.length); - enqueue(write(checkpointId, info, checkBufferType(data)), false); + public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator iterator) { + LOG.debug("add input data, checkpoint id: {}, channel: {}, startSeqNum: {}", checkpointId, info, startSeqNum); + enqueue(write(checkpointId, info, iterator), false); } @Override @@ -168,8 +168,13 @@ private void enqueue(ChannelStateWriteRequest request, boolean atTheFront) { executor.submit(request); } } catch (Exception e) { - request.cancel(e); - throw new RuntimeException("unable to send request to worker", e); + RuntimeException wrapped = new RuntimeException("unable to send request to worker", e); + try { + request.cancel(e); + } catch (Exception cancelException) { + wrapped.addSuppressed(cancelException); + } + throw wrapped; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java index 5de546776a4a2..343c6f488ef9b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/NonSpanningWrapper.java @@ -22,17 +22,21 @@ import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.NextRecordResponse; +import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; +import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; +import org.apache.flink.util.CloseableIterator; import java.io.EOFException; import java.io.IOException; import java.io.UTFDataFormatException; import java.nio.ByteBuffer; -import java.util.Optional; import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER; import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.PARTIAL_RECORD; import static org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.LENGTH_BYTES; +import static org.apache.flink.runtime.io.network.buffer.Buffer.DataType.DATA_BUFFER; final class NonSpanningWrapper implements DataInputView { @@ -69,13 +73,13 @@ void initializeFromMemorySegment(MemorySegment seg, int position, int limit) { this.limit = limit; } - Optional getUnconsumedSegment() { + CloseableIterator getUnconsumedSegment() { if (!hasRemaining()) { - return Optional.empty(); + return CloseableIterator.empty(); } - MemorySegment target = MemorySegmentFactory.allocateUnpooledSegment(remaining()); - segment.copyTo(position, target, 0, remaining()); - return Optional.of(target); + MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(remaining()); + this.segment.copyTo(position, segment, 0, remaining()); + return singleBufferIterator(segment); } boolean hasRemaining() { @@ -359,4 +363,10 @@ private boolean canReadRecord(int recordLength) { return recordLength <= remaining(); } + static CloseableIterator singleBufferIterator(MemorySegment target) { + return CloseableIterator.ofElement( + new NetworkBuffer(target, FreeingBufferRecycler.INSTANCE, DATA_BUFFER, target.size()), + Buffer::recycleBuffer); + } + } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java index 4f4d621632768..07ff5ffcb8f62 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java @@ -20,9 +20,9 @@ import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.util.CloseableIterator; import java.io.IOException; -import java.util.Optional; /** * Interface for turning sequences of memory segments into records. @@ -71,5 +71,5 @@ public boolean isBufferConsumed() { *

Note that the unconsumed buffer might be null if the whole buffer was already consumed * before and there are no partial length or data remained in the end of buffer. */ - Optional getUnconsumedBuffer() throws IOException; + CloseableIterator getUnconsumedBuffer() throws IOException; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java index 430f0db06496c..18ea6cc66a0a2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java @@ -23,6 +23,8 @@ import org.apache.flink.core.memory.DataOutputSerializer; import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.StringUtils; import java.io.BufferedInputStream; @@ -34,11 +36,11 @@ import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.Arrays; -import java.util.Optional; import java.util.Random; import static java.lang.Math.max; import static java.lang.Math.min; +import static org.apache.flink.runtime.io.network.api.serialization.NonSpanningWrapper.singleBufferIterator; import static org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.LENGTH_BYTES; import static org.apache.flink.util.FileUtils.writeCompletely; import static org.apache.flink.util.IOUtils.closeAllQuietly; @@ -165,15 +167,15 @@ private void updateLength(int length) throws IOException { } } - Optional getUnconsumedSegment() throws IOException { + CloseableIterator getUnconsumedSegment() throws IOException { if (isReadingLength()) { - return Optional.of(copyLengthBuffer()); + return singleBufferIterator(copyLengthBuffer()); } else if (isAboveSpillingThreshold()) { throw new UnsupportedOperationException("Unaligned checkpoint currently do not support spilled records."); } else if (recordLength == -1) { - return Optional.empty(); // no remaining partial length or data + return CloseableIterator.empty(); // no remaining partial length or data } else { - return Optional.of(copyDataBuffer()); + return singleBufferIterator(copyDataBuffer()); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java index 75e6b0bfc5587..2d4c24c39fd1c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java @@ -21,18 +21,15 @@ import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.runtime.io.network.buffer.Buffer; -import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; -import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; +import org.apache.flink.util.CloseableIterator; import javax.annotation.concurrent.NotThreadSafe; import java.io.IOException; -import java.util.Optional; import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER; import static org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult.PARTIAL_RECORD; -import static org.apache.flink.runtime.io.network.buffer.Buffer.DataType.DATA_BUFFER; /** * @param The type of the record to be deserialized. @@ -76,14 +73,8 @@ public Buffer getCurrentBuffer () { } @Override - public Optional getUnconsumedBuffer() throws IOException { - final Optional unconsumedSegment; - if (nonSpanningWrapper.hasRemaining()) { - unconsumedSegment = nonSpanningWrapper.getUnconsumedSegment(); - } else { - unconsumedSegment = spanningWrapper.getUnconsumedSegment(); - } - return unconsumedSegment.map(segment -> new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, DATA_BUFFER, segment.size())); + public CloseableIterator getUnconsumedBuffer() throws IOException { + return nonSpanningWrapper.hasRemaining() ? nonSpanningWrapper.getUnconsumedSegment() : spanningWrapper.getUnconsumedSegment(); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java index 6db81e9f79091..4e1f260d4f54d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java @@ -31,6 +31,7 @@ import org.apache.flink.runtime.io.network.buffer.BufferReceivedListener; import org.apache.flink.runtime.io.network.partition.PartitionNotFoundException; import org.apache.flink.runtime.io.network.partition.ResultPartitionID; +import org.apache.flink.util.CloseableIterator; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; @@ -207,7 +208,7 @@ public void spillInflightBuffers(long checkpointId, ChannelStateWriter channelSt checkpointId, channelInfo, ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN, - inflightBuffers.toArray(new Buffer[0])); + CloseableIterator.fromList(inflightBuffers, Buffer::recycleBuffer)); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherTest.java index f953c22d14f8a..00c8ca75d1cbc 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestDispatcherTest.java @@ -17,8 +17,13 @@ package org.apache.flink.runtime.checkpoint.channel; +import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter.ChannelStateWriteResult; +import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; +import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; import org.apache.flink.runtime.state.CheckpointStorageLocationReference; +import org.apache.flink.util.CloseableIterator; import org.junit.Test; import org.junit.runner.RunWith; @@ -83,7 +88,10 @@ private static CheckpointInProgressRequest completeIn() { } private static ChannelStateWriteRequest writeIn() { - return write(CHECKPOINT_ID, new InputChannelInfo(1, 1)); + return write(CHECKPOINT_ID, new InputChannelInfo(1, 1), CloseableIterator.ofElement( + new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(1), FreeingBufferRecycler.INSTANCE), + Buffer::recycleBuffer + )); } private static ChannelStateWriteRequest writeOut() { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java index 5aad9c6dcecae..a299b34dc4c9e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java @@ -30,7 +30,6 @@ import static org.apache.flink.runtime.checkpoint.channel.ChannelStateWriteRequestDispatcher.NO_OP; import static org.apache.flink.util.ExceptionUtils.findThrowable; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java index 44552e6e74668..92a7e881f57a5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java @@ -35,6 +35,7 @@ import java.util.function.Consumer; import static org.apache.flink.runtime.state.ChannelPersistenceITCase.getStreamFactoryFactory; +import static org.apache.flink.util.CloseableIterator.ofElements; import static org.apache.flink.util.ExceptionUtils.findThrowable; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -47,14 +48,16 @@ public class ChannelStateWriterImplTest { private static final long CHECKPOINT_ID = 42L; @Test(expected = IllegalArgumentException.class) - public void testAddEventBuffer() { + public void testAddEventBuffer() throws Exception { + NetworkBuffer dataBuf = getBuffer(); NetworkBuffer eventBuf = getBuffer(); eventBuf.setDataType(Buffer.DataType.EVENT_BUFFER); - ChannelStateWriterImpl writer = openWriter(); - callStart(writer); try { - writer.addInputData(CHECKPOINT_ID, new InputChannelInfo(1, 1), 1, eventBuf, dataBuf); + runWithSyncWorker(writer -> { + callStart(writer); + writer.addInputData(CHECKPOINT_ID, new InputChannelInfo(1, 1), 1, ofElements(Buffer::recycleBuffer, eventBuf, dataBuf)); + }); } finally { assertTrue(dataBuf.isRecycled()); } @@ -285,7 +288,7 @@ private void callStart(ChannelStateWriter writer) { } private void callAddInputData(ChannelStateWriter writer, NetworkBuffer... buffer) { - writer.addInputData(CHECKPOINT_ID, new InputChannelInfo(1, 1), 1, buffer); + writer.addInputData(CHECKPOINT_ID, new InputChannelInfo(1, 1), 1, ofElements(Buffer::recycleBuffer, buffer)); } private void callAbort(ChannelStateWriter writer) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/CheckpointInProgressRequestTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/CheckpointInProgressRequestTest.java index 3617b8f95d0b3..556bf49d9b32a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/CheckpointInProgressRequestTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/CheckpointInProgressRequestTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * {@link CheckpointInProgressRequest} test. @@ -41,7 +42,11 @@ public void testNoCancelTwice() throws Exception { Thread[] threads = new Thread[barrier.getParties()]; for (int i = 0; i < barrier.getParties(); i++) { threads[i] = new Thread(() -> { - request.cancel(new RuntimeException("test")); + try { + request.cancel(new RuntimeException("test")); + } catch (Exception e) { + fail(e.getMessage()); + } await(barrier); }); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/MockChannelStateWriter.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/MockChannelStateWriter.java index 5dcc00cc10137..0a61066d097ee 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/MockChannelStateWriter.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/MockChannelStateWriter.java @@ -19,6 +19,9 @@ import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.util.CloseableIterator; + +import static org.apache.flink.util.ExceptionUtils.rethrow; /** * A no op implementation that performs basic checks of the contract, but does not actually write any data. @@ -49,10 +52,12 @@ public void start(long checkpointId, CheckpointOptions checkpointOptions) { } @Override - public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, Buffer... data) { + public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator iterator) { checkCheckpointId(checkpointId); - for (final Buffer buffer : data) { - buffer.recycleBuffer(); + try { + iterator.close(); + } catch (Exception e) { + rethrow(e); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecordingChannelStateWriter.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecordingChannelStateWriter.java index b53e37b722ce9..d0cfe3f2cb8a2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecordingChannelStateWriter.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecordingChannelStateWriter.java @@ -19,12 +19,15 @@ import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.shaded.guava18.com.google.common.collect.LinkedListMultimap; import org.apache.flink.shaded.guava18.com.google.common.collect.ListMultimap; import java.util.Arrays; +import static org.apache.flink.util.ExceptionUtils.rethrow; + /** * A simple {@link ChannelStateWriter} used to write unit tests. */ @@ -54,9 +57,14 @@ public void start(long checkpointId, CheckpointOptions checkpointOptions) { } @Override - public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, Buffer... data) { + public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator iterator) { checkCheckpointId(checkpointId); - addedInput.putAll(info, Arrays.asList(data)); + iterator.forEachRemaining(b -> addedInput.put(info, b)); + try { + iterator.close(); + } catch (Exception e) { + rethrow(e); + } } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializationTest.java index 183df10cc7b56..e35b3111cc8ed 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializationTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializationTest.java @@ -30,6 +30,7 @@ import org.apache.flink.testutils.serialization.types.SerializationTestType; import org.apache.flink.testutils.serialization.types.SerializationTestTypeFactory; import org.apache.flink.testutils.serialization.types.Util; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.TestLogger; import org.junit.Assert; @@ -46,7 +47,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Random; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSingleBuffer; @@ -293,14 +293,15 @@ private static Buffer appendLeftOverBytes(Buffer buffer, byte[] leftOverBytes) { } } - private static void assertUnconsumedBuffer(ByteArrayOutputStream expected, Optional actual) { - if (!actual.isPresent()) { + private static void assertUnconsumedBuffer(ByteArrayOutputStream expected, CloseableIterator actual) throws Exception { + if (!actual.hasNext()) { Assert.assertEquals(expected.size(), 0); } ByteBuffer expectedByteBuffer = ByteBuffer.wrap(expected.toByteArray()); - ByteBuffer actualByteBuffer = actual.get().getNioBufferReadable(); + ByteBuffer actualByteBuffer = actual.next().getNioBufferReadable(); Assert.assertEquals(expectedByteBuffer, actualByteBuffer); + actual.close(); } private static void writeBuffer(ByteBuffer buffer, OutputStream stream) throws IOException { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java index abcf563b0bf39..6f49b445e9f1e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java @@ -66,6 +66,7 @@ import org.apache.flink.runtime.shuffle.ShuffleDescriptor; import org.apache.flink.runtime.shuffle.UnknownShuffleDescriptor; import org.apache.flink.runtime.state.CheckpointStorageLocationReference; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.ExceptionUtils; import org.junit.Test; @@ -74,7 +75,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -94,6 +94,7 @@ import static org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannelTest.submitTasksAndWaitForResults; import static org.apache.flink.runtime.io.network.util.TestBufferFactory.createBuffer; import static org.apache.flink.runtime.util.NettyShuffleDescriptorBuilder.createRemoteWithIdAndLocation; +import static org.apache.flink.util.ExceptionUtils.rethrow; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -989,8 +990,15 @@ public void notifyBarrierReceived(CheckpointBarrier barrier, InputChannelInfo ch inputChannel.spillInflightBuffers(0, new ChannelStateWriterImpl.NoOpChannelStateWriter() { @Override - public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, Buffer... data) { - inflightBuffers.addAll(Arrays.asList(data)); + public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator iterator) { + List list = new ArrayList<>(); + iterator.forEachRemaining(list::add); + inflightBuffers.addAll(list); + try { + iterator.close(); + } catch (Exception e) { + rethrow(e); + } } }); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ChannelPersistenceITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ChannelPersistenceITCase.java index 3f5e2cc00101a..a77dbbfd32553 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ChannelPersistenceITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ChannelPersistenceITCase.java @@ -50,6 +50,7 @@ import static org.apache.flink.runtime.checkpoint.CheckpointType.CHECKPOINT; import static org.apache.flink.runtime.checkpoint.channel.ChannelStateReader.ReadResult.NO_MORE_DATA; import static org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN; +import static org.apache.flink.util.CloseableIterator.ofElements; import static org.apache.flink.util.Preconditions.checkState; import static org.junit.Assert.assertArrayEquals; @@ -102,7 +103,7 @@ private ChannelStateWriteResult write(long checkpointId, Map e : icBuffers.entrySet()) { - writer.addInputData(checkpointId, e.getKey(), SEQUENCE_NUMBER_UNKNOWN, e.getValue()); + writer.addInputData(checkpointId, e.getKey(), SEQUENCE_NUMBER_UNKNOWN, ofElements(Buffer::recycleBuffer, e.getValue())); } writer.finishInput(checkpointId); for (Map.Entry e : rsBuffers.entrySet()) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierUnaligner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierUnaligner.java index f98e83fe2ae39..d39accf97ec11 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierUnaligner.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierUnaligner.java @@ -45,6 +45,7 @@ import java.util.function.Function; import java.util.stream.IntStream; +import static org.apache.flink.util.CloseableIterator.ofElement; import static org.apache.flink.util.Preconditions.checkNotNull; /** @@ -330,7 +331,7 @@ public synchronized void notifyBufferReceived(Buffer buffer, InputChannelInfo ch currentReceivedCheckpointId, channelInfo, ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN, - buffer); + ofElement(buffer, Buffer::recycleBuffer)); } else { buffer.recycleBuffer(); } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java index 07826c7eab37c..6723a2da52f14 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java @@ -212,12 +212,11 @@ public CompletableFuture prepareSnapshot( // Assumption for retrieving buffers = one concurrent checkpoint RecordDeserializer deserializer = recordDeserializers[channelIndex]; if (deserializer != null) { - deserializer.getUnconsumedBuffer().ifPresent(buffer -> - channelStateWriter.addInputData( - checkpointId, - channel.getChannelInfo(), - ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN, - buffer)); + channelStateWriter.addInputData( + checkpointId, + channel.getChannelInfo(), + ChannelStateWriter.SEQUENCE_NUMBER_UNKNOWN, + deserializer.getUnconsumedBuffer()); } checkpointedInputGate.spillInflightBuffers(checkpointId, channelIndex, channelStateWriter); From 4e323c33a9f25673587ee0e8f4f9786b21db666c Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Thu, 7 May 2020 15:30:56 +0200 Subject: [PATCH 055/773] [FLINK-17547][task][hotfix] Extract RefCountedFileWithStream from RefCountedFile Motivation: use RefCountedFile for reading as well. --- .../flink/fs/s3/common/FlinkS3FileSystem.java | 4 +- .../utils/RefCountedBufferingFileStream.java | 10 +- .../fs/s3/common/utils/RefCountedFile.java | 59 +---------- .../utils/RefCountedFileWithStream.java | 92 ++++++++++++++++ .../utils/RefCountedTmpFileCreator.java | 10 +- .../S3RecoverableFsDataOutputStream.java | 12 +-- .../S3RecoverableMultipartUploadFactory.java | 8 +- .../s3/common/writer/S3RecoverableWriter.java | 8 +- .../RefCountedBufferingFileStreamTest.java | 4 +- .../s3/common/utils/RefCountedFileTest.java | 59 ++--------- .../utils/RefCountedFileWithStreamTest.java | 100 ++++++++++++++++++ .../RecoverableMultiPartUploadImplTest.java | 4 +- .../S3RecoverableFsDataOutputStreamTest.java | 10 +- 13 files changed, 237 insertions(+), 143 deletions(-) create mode 100644 flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java create mode 100644 flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStreamTest.java diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java index 5248e061a12ec..3514bbcb4e36a 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/FlinkS3FileSystem.java @@ -21,7 +21,7 @@ import org.apache.flink.core.fs.EntropyInjectingFileSystem; import org.apache.flink.core.fs.FileSystemKind; import org.apache.flink.core.fs.RecoverableWriter; -import org.apache.flink.fs.s3.common.utils.RefCountedFile; +import org.apache.flink.fs.s3.common.utils.RefCountedFileWithStream; import org.apache.flink.fs.s3.common.utils.RefCountedTmpFileCreator; import org.apache.flink.fs.s3.common.writer.S3AccessHelper; import org.apache.flink.fs.s3.common.writer.S3RecoverableWriter; @@ -57,7 +57,7 @@ public class FlinkS3FileSystem extends HadoopFileSystem implements EntropyInject private final String localTmpDir; - private final FunctionWithException tmpFileCreator; + private final FunctionWithException tmpFileCreator; @Nullable private final S3AccessHelper s3AccessHelper; diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java index 29f2590803cc8..5f149df6c87c1 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStream.java @@ -29,7 +29,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull; /** - * A {@link RefCountedFile} that also uses an in-memory buffer for buffering small writes. + * A {@link RefCountedFileWithStream} that also uses an in-memory buffer for buffering small writes. * This is done to avoid frequent 'flushes' of the file stream to disk. */ @Internal @@ -37,7 +37,7 @@ public class RefCountedBufferingFileStream extends RefCountedFSOutputStream { public static final int BUFFER_SIZE = 4096; - private final RefCountedFile currentTmpFile; + private final RefCountedFileWithStream currentTmpFile; /** The write buffer. */ private final byte[] buffer; @@ -49,7 +49,7 @@ public class RefCountedBufferingFileStream extends RefCountedFSOutputStream { @VisibleForTesting public RefCountedBufferingFileStream( - final RefCountedFile file, + final RefCountedFileWithStream file, final int bufferSize) { checkArgument(bufferSize > 0L); @@ -165,7 +165,7 @@ public int getReferenceCounter() { // ------------------------- Factory Methods ------------------------- public static RefCountedBufferingFileStream openNew( - final FunctionWithException tmpFileProvider) throws IOException { + final FunctionWithException tmpFileProvider) throws IOException { return new RefCountedBufferingFileStream( tmpFileProvider.apply(null), @@ -173,7 +173,7 @@ public static RefCountedBufferingFileStream openNew( } public static RefCountedBufferingFileStream restore( - final FunctionWithException tmpFileProvider, + final FunctionWithException tmpFileProvider, final File initialTmpFile) throws IOException { return new RefCountedBufferingFileStream( diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java index 178763631e1bd..9675f09a1fa91 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java @@ -21,11 +21,10 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.IOUtils; +import org.apache.flink.util.RefCounted; import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.nio.file.Files; import java.util.concurrent.atomic.AtomicInteger; @@ -40,21 +39,13 @@ public class RefCountedFile implements RefCounted { private final File file; - private final OffsetAwareOutputStream stream; - private final AtomicInteger references; - private boolean closed; + protected boolean closed; - private RefCountedFile( - final File file, - final OutputStream currentOut, - final long bytesInCurrentPart) { + protected RefCountedFile(final File file) { this.file = checkNotNull(file); this.references = new AtomicInteger(1); - this.stream = new OffsetAwareOutputStream( - currentOut, - bytesInCurrentPart); this.closed = false; } @@ -62,33 +53,6 @@ public File getFile() { return file; } - public OffsetAwareOutputStream getStream() { - return stream; - } - - public long getLength() { - return stream.getLength(); - } - - public void write(byte[] b, int off, int len) throws IOException { - requireOpened(); - if (len > 0) { - stream.write(b, off, len); - } - } - - public void flush() throws IOException { - requireOpened(); - stream.flush(); - } - - public void closeStream() { - if (!closed) { - IOUtils.closeQuietly(stream); - closed = true; - } - } - @Override public void retain() { references.incrementAndGet(); @@ -119,22 +83,7 @@ private void requireOpened() throws IOException { } @VisibleForTesting - int getReferenceCounter() { + public int getReferenceCounter() { return references.get(); } - - // ------------------------------ Factory methods for initializing a temporary file ------------------------------ - - public static RefCountedFile newFile( - final File file, - final OutputStream currentOut) throws IOException { - return new RefCountedFile(file, currentOut, 0L); - } - - public static RefCountedFile restoredFile( - final File file, - final OutputStream currentOut, - final long bytesInCurrentPart) { - return new RefCountedFile(file, currentOut, bytesInCurrentPart); - } } diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java new file mode 100644 index 0000000000000..94b8527adcf65 --- /dev/null +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.fs.s3.common.utils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.IOUtils; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A reference counted file which is deleted as soon as no caller + * holds a reference to the wrapped {@link File}. + */ +@Internal +public class RefCountedFileWithStream extends RefCountedFile { + + private final OffsetAwareOutputStream stream; + + private RefCountedFileWithStream( + final File file, + final OutputStream currentOut, + final long bytesInCurrentPart) { + super(file); + this.stream = new OffsetAwareOutputStream(currentOut, bytesInCurrentPart); + } + + public OffsetAwareOutputStream getStream() { + return stream; + } + + public long getLength() { + return stream.getLength(); + } + + public void write(byte[] b, int off, int len) throws IOException { + requireOpened(); + if (len > 0) { + stream.write(b, off, len); + } + } + + void flush() throws IOException { + requireOpened(); + stream.flush(); + } + + void closeStream() { + if (!closed) { + IOUtils.closeQuietly(stream); + closed = true; + } + } + + private void requireOpened() throws IOException { + if (closed) { + throw new IOException("Stream closed."); + } + } + + // ------------------------------ Factory methods for initializing a temporary file ------------------------------ + + public static RefCountedFileWithStream newFile( + final File file, + final OutputStream currentOut) throws IOException { + return new RefCountedFileWithStream(file, currentOut, 0L); + } + + public static RefCountedFileWithStream restoredFile( + final File file, + final OutputStream currentOut, + final long bytesInCurrentPart) { + return new RefCountedFileWithStream(file, currentOut, bytesInCurrentPart); + } +} diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedTmpFileCreator.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedTmpFileCreator.java index 7a928d0c99aa5..51b417c42ccbd 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedTmpFileCreator.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedTmpFileCreator.java @@ -34,10 +34,10 @@ import static org.apache.flink.util.Preconditions.checkArgument; /** - * A utility class that creates local {@link RefCountedFile reference counted files} that serve as temporary files. + * A utility class that creates local {@link RefCountedFileWithStream reference counted files} that serve as temporary files. */ @Internal -public class RefCountedTmpFileCreator implements FunctionWithException { +public class RefCountedTmpFileCreator implements FunctionWithException { private final File[] tempDirectories; @@ -70,7 +70,7 @@ private RefCountedTmpFileCreator(File... tempDirectories) { * @throws IOException Thrown, if the stream to the temp file could not be opened. */ @Override - public RefCountedFile apply(File file) throws IOException { + public RefCountedFileWithStream apply(File file) throws IOException { final File directory = tempDirectories[nextIndex()]; while (true) { @@ -78,10 +78,10 @@ public RefCountedFile apply(File file) throws IOException { if (file == null) { final File newFile = new File(directory, ".tmp_" + UUID.randomUUID()); final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); - return RefCountedFile.newFile(newFile, out); + return RefCountedFileWithStream.newFile(newFile, out); } else { final OutputStream out = Files.newOutputStream(file.toPath(), StandardOpenOption.APPEND); - return RefCountedFile.restoredFile(file, out, file.length()); + return RefCountedFileWithStream.restoredFile(file, out, file.length()); } } catch (FileAlreadyExistsException ignored) { // fall through the loop and retry diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStream.java index 220ddd58eb989..5447026be94ba 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStream.java @@ -23,7 +23,7 @@ import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream; import org.apache.flink.fs.s3.common.utils.RefCountedFSOutputStream; -import org.apache.flink.fs.s3.common.utils.RefCountedFile; +import org.apache.flink.fs.s3.common.utils.RefCountedFileWithStream; import org.apache.flink.util.function.FunctionWithException; import org.apache.commons.io.IOUtils; @@ -60,7 +60,7 @@ public final class S3RecoverableFsDataOutputStream extends RecoverableFsDataOutp private final RecoverableMultiPartUpload upload; - private final FunctionWithException tmpFileProvider; + private final FunctionWithException tmpFileProvider; /** * The number of bytes at which we start a new part of the multipart upload. @@ -80,7 +80,7 @@ public final class S3RecoverableFsDataOutputStream extends RecoverableFsDataOutp */ S3RecoverableFsDataOutputStream( RecoverableMultiPartUpload upload, - FunctionWithException tempFileCreator, + FunctionWithException tempFileCreator, RefCountedFSOutputStream initialTmpFile, long userDefinedMinPartSize, long bytesBeforeCurrentPart) { @@ -228,7 +228,7 @@ private void unlock() { public static S3RecoverableFsDataOutputStream newStream( final RecoverableMultiPartUpload upload, - final FunctionWithException tmpFileCreator, + final FunctionWithException tmpFileCreator, final long userDefinedMinPartSize) throws IOException { checkArgument(userDefinedMinPartSize >= S3_MULTIPART_MIN_PART_SIZE); @@ -245,7 +245,7 @@ public static S3RecoverableFsDataOutputStream newStream( public static S3RecoverableFsDataOutputStream recoverStream( final RecoverableMultiPartUpload upload, - final FunctionWithException tmpFileCreator, + final FunctionWithException tmpFileCreator, final long userDefinedMinPartSize, final long bytesBeforeCurrentPart) throws IOException { @@ -264,7 +264,7 @@ public static S3RecoverableFsDataOutputStream recoverStream( } private static RefCountedBufferingFileStream boundedBufferingFileStream( - final FunctionWithException tmpFileCreator, + final FunctionWithException tmpFileCreator, final Optional incompletePart) throws IOException { if (!incompletePart.isPresent()) { diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java index 3727e25790437..b7fb8fb9bdbd6 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableMultipartUploadFactory.java @@ -21,7 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.fs.Path; import org.apache.flink.fs.s3.common.utils.BackPressuringExecutor; -import org.apache.flink.fs.s3.common.utils.RefCountedFile; +import org.apache.flink.fs.s3.common.utils.RefCountedFileWithStream; import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.FunctionWithException; @@ -43,7 +43,7 @@ final class S3RecoverableMultipartUploadFactory { private final S3AccessHelper s3AccessHelper; - private final FunctionWithException tmpFileSupplier; + private final FunctionWithException tmpFileSupplier; private final int maxConcurrentUploadsPerStream; @@ -54,7 +54,7 @@ final class S3RecoverableMultipartUploadFactory { final S3AccessHelper s3AccessHelper, final int maxConcurrentUploadsPerStream, final Executor executor, - final FunctionWithException tmpFileSupplier) { + final FunctionWithException tmpFileSupplier) { this.fs = Preconditions.checkNotNull(fs); this.maxConcurrentUploadsPerStream = maxConcurrentUploadsPerStream; @@ -92,7 +92,7 @@ private Optional recoverInProgressPart(S3Recoverable recoverable) throws I } // download the file (simple way) - final RefCountedFile refCountedFile = tmpFileSupplier.apply(null); + final RefCountedFileWithStream refCountedFile = tmpFileSupplier.apply(null); final File file = refCountedFile.getFile(); final long numBytes = s3AccessHelper.getObject(objectKey, file); diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java index ddb4443c58564..a6b62cca304c8 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/writer/S3RecoverableWriter.java @@ -25,7 +25,7 @@ import org.apache.flink.core.fs.RecoverableFsDataOutputStream.Committer; import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.core.io.SimpleVersionedSerializer; -import org.apache.flink.fs.s3.common.utils.RefCountedFile; +import org.apache.flink.fs.s3.common.utils.RefCountedFileWithStream; import org.apache.flink.util.function.FunctionWithException; import org.apache.hadoop.fs.FileSystem; @@ -50,7 +50,7 @@ @PublicEvolving public class S3RecoverableWriter implements RecoverableWriter { - private final FunctionWithException tempFileCreator; + private final FunctionWithException tempFileCreator; private final long userDefinedMinPartSize; @@ -62,7 +62,7 @@ public class S3RecoverableWriter implements RecoverableWriter { S3RecoverableWriter( final S3AccessHelper s3AccessHelper, final S3RecoverableMultipartUploadFactory uploadFactory, - final FunctionWithException tempFileCreator, + final FunctionWithException tempFileCreator, final long userDefinedMinPartSize) { this.s3AccessHelper = checkNotNull(s3AccessHelper); @@ -144,7 +144,7 @@ private static S3Recoverable castToS3Recoverable(CommitRecoverable recoverable) public static S3RecoverableWriter writer( final FileSystem fs, - final FunctionWithException tempFileCreator, + final FunctionWithException tempFileCreator, final S3AccessHelper s3AccessHelper, final Executor uploadThreadPool, final long userDefinedMinPartSize, diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java index 50ea9bd64a6e1..368c9cfe0ff47 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedBufferingFileStreamTest.java @@ -134,11 +134,11 @@ private RefCountedBufferingFileStream getStreamToTest() throws IOException { return new RefCountedBufferingFileStream(getRefCountedFileWithContent(), BUFFER_SIZE); } - private RefCountedFile getRefCountedFileWithContent() throws IOException { + private RefCountedFileWithStream getRefCountedFileWithContent() throws IOException { final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID()); final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); - return RefCountedFile.newFile(newFile, out); + return RefCountedFileWithStream.newFile(newFile, out); } private static byte[] bytesOf(String str) { diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java index 2e03197142a20..217f4e163ecd0 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java @@ -25,14 +25,14 @@ import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.UUID; import java.util.stream.Stream; +import static org.apache.flink.util.Preconditions.checkState; + /** * Tests for the {@link RefCountedFile}. */ @@ -44,9 +44,9 @@ public class RefCountedFileTest { @Test public void releaseToZeroRefCounterShouldDeleteTheFile() throws IOException { final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID()); - final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); + checkState(newFile.createNewFile()); - RefCountedFile fileUnderTest = RefCountedFile.newFile(newFile, out); + RefCountedFile fileUnderTest = new RefCountedFile(newFile); verifyTheFileIsStillThere(); fileUnderTest.release(); @@ -59,10 +59,10 @@ public void releaseToZeroRefCounterShouldDeleteTheFile() throws IOException { @Test public void retainsShouldRequirePlusOneReleasesToDeleteTheFile() throws IOException { final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID()); - final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); + checkState(newFile.createNewFile()); // the reference counter always starts with 1 (not 0). This is why we need +1 releases - RefCountedFile fileUnderTest = RefCountedFile.newFile(newFile, out); + RefCountedFile fileUnderTest = new RefCountedFile(newFile); verifyTheFileIsStillThere(); fileUnderTest.retain(); @@ -85,59 +85,12 @@ public void retainsShouldRequirePlusOneReleasesToDeleteTheFile() throws IOExcept } } - @Test - public void writeShouldSucceed() throws IOException { - byte[] content = bytesOf("hello world"); - - final RefCountedFile fileUnderTest = getClosedRefCountedFileWithContent(content); - long fileLength = fileUnderTest.getLength(); - - Assert.assertEquals(content.length, fileLength); - } - - @Test - public void closeShouldNotReleaseReference() throws IOException { - getClosedRefCountedFileWithContent("hello world"); - verifyTheFileIsStillThere(); - } - - @Test(expected = IOException.class) - public void writeAfterCloseShouldThrowException() throws IOException { - final RefCountedFile fileUnderTest = getClosedRefCountedFileWithContent("hello world"); - byte[] content = bytesOf("Hello Again"); - fileUnderTest.write(content, 0, content.length); - } - - @Test(expected = IOException.class) - public void flushAfterCloseShouldThrowException() throws IOException { - final RefCountedFile fileUnderTest = getClosedRefCountedFileWithContent("hello world"); - fileUnderTest.flush(); - } - - // ------------------------------------- Utilities ------------------------------------- - private void verifyTheFileIsStillThere() throws IOException { try (Stream files = Files.list(temporaryFolder.getRoot().toPath())) { Assert.assertEquals(1L, files.count()); } } - private RefCountedFile getClosedRefCountedFileWithContent(String content) throws IOException { - return getClosedRefCountedFileWithContent(bytesOf(content)); - } - - private RefCountedFile getClosedRefCountedFileWithContent(byte[] content) throws IOException { - final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID()); - final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); - - final RefCountedFile fileUnderTest = RefCountedFile.newFile(newFile, out); - - fileUnderTest.write(content, 0, content.length); - - fileUnderTest.closeStream(); - return fileUnderTest; - } - private static byte[] bytesOf(String str) { return str.getBytes(StandardCharsets.UTF_8); } diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStreamTest.java new file mode 100644 index 0000000000000..7aa7240ff6b2f --- /dev/null +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStreamTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.fs.s3.common.utils; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * Tests for the {@link RefCountedFileWithStream}. + */ +public class RefCountedFileWithStreamTest { + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void writeShouldSucceed() throws IOException { + byte[] content = bytesOf("hello world"); + + final RefCountedFileWithStream fileUnderTest = getClosedRefCountedFileWithContent(content); + long fileLength = fileUnderTest.getLength(); + + Assert.assertEquals(content.length, fileLength); + } + + @Test + public void closeShouldNotReleaseReference() throws IOException { + getClosedRefCountedFileWithContent("hello world"); + verifyTheFileIsStillThere(); + } + + @Test(expected = IOException.class) + public void writeAfterCloseShouldThrowException() throws IOException { + final RefCountedFileWithStream fileUnderTest = getClosedRefCountedFileWithContent("hello world"); + byte[] content = bytesOf("Hello Again"); + fileUnderTest.write(content, 0, content.length); + } + + @Test(expected = IOException.class) + public void flushAfterCloseShouldThrowException() throws IOException { + final RefCountedFileWithStream fileUnderTest = getClosedRefCountedFileWithContent("hello world"); + fileUnderTest.flush(); + } + + // ------------------------------------- Utilities ------------------------------------- + + private void verifyTheFileIsStillThere() throws IOException { + try (Stream files = Files.list(temporaryFolder.getRoot().toPath())) { + Assert.assertEquals(1L, files.count()); + } + } + + private RefCountedFileWithStream getClosedRefCountedFileWithContent(String content) throws IOException { + return getClosedRefCountedFileWithContent(bytesOf(content)); + } + + private RefCountedFileWithStream getClosedRefCountedFileWithContent(byte[] content) throws IOException { + final File newFile = new File(temporaryFolder.getRoot(), ".tmp_" + UUID.randomUUID()); + final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); + + final RefCountedFileWithStream fileUnderTest = RefCountedFileWithStream.newFile(newFile, out); + + fileUnderTest.write(content, 0, content.length); + + fileUnderTest.closeStream(); + return fileUnderTest; + } + + private static byte[] bytesOf(String str) { + return str.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java index f01da886c24ef..e8c6e9e8e74a2 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/RecoverableMultiPartUploadImplTest.java @@ -19,7 +19,7 @@ package org.apache.flink.fs.s3.common.writer; import org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream; -import org.apache.flink.fs.s3.common.utils.RefCountedFile; +import org.apache.flink.fs.s3.common.utils.RefCountedFileWithStream; import org.apache.flink.util.IOUtils; import org.apache.flink.util.MathUtils; @@ -320,7 +320,7 @@ private RefCountedBufferingFileStream writeContent(byte[] content) throws IOExce final OutputStream out = Files.newOutputStream(newFile.toPath(), StandardOpenOption.CREATE_NEW); final RefCountedBufferingFileStream testStream = - new RefCountedBufferingFileStream(RefCountedFile.newFile(newFile, out), BUFFER_SIZE); + new RefCountedBufferingFileStream(RefCountedFileWithStream.newFile(newFile, out), BUFFER_SIZE); testStream.write(content, 0, content.length); return testStream; diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java index 14ed2e294f7d2..b7c94c4b30b68 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java +++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/writer/S3RecoverableFsDataOutputStreamTest.java @@ -22,7 +22,7 @@ import org.apache.flink.core.fs.RecoverableWriter; import org.apache.flink.fs.s3.common.utils.RefCountedBufferingFileStream; import org.apache.flink.fs.s3.common.utils.RefCountedFSOutputStream; -import org.apache.flink.fs.s3.common.utils.RefCountedFile; +import org.apache.flink.fs.s3.common.utils.RefCountedFileWithStream; import org.apache.flink.util.MathUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.FunctionWithException; @@ -483,7 +483,7 @@ public String toString() { } } - private static class TestFileProvider implements FunctionWithException { + private static class TestFileProvider implements FunctionWithException { private final TemporaryFolder folder; @@ -492,16 +492,16 @@ private static class TestFileProvider implements FunctionWithException Date: Thu, 7 May 2020 15:31:51 +0200 Subject: [PATCH 056/773] [FLINK-17547][task][hotfix] Move RefCountedFile to flink-core to use it in SpanningWrapper --- .../main/java/org/apache/flink/core/fs}/RefCountedFile.java | 4 ++-- .../src/main/java/org/apache/flink/util}/RefCounted.java | 2 +- .../java/org/apache/flink/core/fs}/RefCountedFileTest.java | 2 +- .../flink/fs/s3/common/utils/RefCountedFSOutputStream.java | 1 + .../flink/fs/s3/common/utils/RefCountedFileWithStream.java | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) rename {flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils => flink-core/src/main/java/org/apache/flink/core/fs}/RefCountedFile.java (96%) rename {flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils => flink-core/src/main/java/org/apache/flink/util}/RefCounted.java (96%) rename {flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils => flink-core/src/test/java/org/apache/flink/core/fs}/RefCountedFileTest.java (98%) diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java b/flink-core/src/main/java/org/apache/flink/core/fs/RefCountedFile.java similarity index 96% rename from flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java rename to flink-core/src/main/java/org/apache/flink/core/fs/RefCountedFile.java index 9675f09a1fa91..7cbc47f20bdc2 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFile.java +++ b/flink-core/src/main/java/org/apache/flink/core/fs/RefCountedFile.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.fs.s3.common.utils; +package org.apache.flink.core.fs; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; @@ -43,7 +43,7 @@ public class RefCountedFile implements RefCounted { protected boolean closed; - protected RefCountedFile(final File file) { + public RefCountedFile(final File file) { this.file = checkNotNull(file); this.references = new AtomicInteger(1); this.closed = false; diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCounted.java b/flink-core/src/main/java/org/apache/flink/util/RefCounted.java similarity index 96% rename from flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCounted.java rename to flink-core/src/main/java/org/apache/flink/util/RefCounted.java index 84b0fa086117e..33496a00f01fa 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCounted.java +++ b/flink-core/src/main/java/org/apache/flink/util/RefCounted.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.fs.s3.common.utils; +package org.apache.flink.util; import org.apache.flink.annotation.Internal; diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/RefCountedFileTest.java similarity index 98% rename from flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java rename to flink-core/src/test/java/org/apache/flink/core/fs/RefCountedFileTest.java index 217f4e163ecd0..58ca29aaee6fc 100644 --- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/utils/RefCountedFileTest.java +++ b/flink-core/src/test/java/org/apache/flink/core/fs/RefCountedFileTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.fs.s3.common.utils; +package org.apache.flink.core.fs; import org.junit.Assert; import org.junit.Rule; diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java index d51e37e8bfbf3..a36175df62b48 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFSOutputStream.java @@ -20,6 +20,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.core.fs.FSDataOutputStream; +import org.apache.flink.util.RefCounted; import java.io.File; import java.io.IOException; diff --git a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java index 94b8527adcf65..bcb0057983d4b 100644 --- a/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java +++ b/flink-filesystems/flink-s3-fs-base/src/main/java/org/apache/flink/fs/s3/common/utils/RefCountedFileWithStream.java @@ -19,6 +19,7 @@ package org.apache.flink.fs.s3.common.utils; import org.apache.flink.annotation.Internal; +import org.apache.flink.core.fs.RefCountedFile; import org.apache.flink.util.IOUtils; import java.io.File; From 3dacffe35709f9747923dad4c7028baec27e2651 Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Thu, 7 May 2020 15:59:26 +0200 Subject: [PATCH 057/773] [FLINK-17547][task] Use RefCountedFile in SpanningWrapper (todo: merge with next?) --- .../api/serialization/SpanningWrapper.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java index 18ea6cc66a0a2..9cffde37c10d7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java @@ -17,6 +17,7 @@ package org.apache.flink.runtime.io.network.api.serialization; +import org.apache.flink.core.fs.RefCountedFile; import org.apache.flink.core.memory.DataInputDeserializer; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataInputViewStreamWrapper; @@ -74,7 +75,7 @@ final class SpanningWrapper { private int leftOverLimit; - private File spillFile; + private RefCountedFile spillFile; private DataInputViewStreamWrapper spillFileReader; @@ -136,7 +137,7 @@ private void copyIntoFile(MemorySegment segment, int offset, int length) throws accumulatedRecordBytes += length; if (hasFullRecord()) { spillingChannel.close(); - spillFileReader = new DataInputViewStreamWrapper(new BufferedInputStream(new FileInputStream(spillFile), FILE_BUFFER_SIZE)); + spillFileReader = new DataInputViewStreamWrapper(new BufferedInputStream(new FileInputStream(spillFile.getFile()), FILE_BUFFER_SIZE)); } } @@ -220,7 +221,6 @@ int getNumGatheredBytes() { return accumulatedRecordBytes + (recordLength >= 0 ? LENGTH_BYTES : lengthBuffer.position()); } - @SuppressWarnings("ResultOfMethodCallIgnored") public void clear() { buffer = initialBuffer; serializationReadBuffer.releaseArrays(); @@ -232,7 +232,7 @@ public void clear() { leftOverLimit = 0; accumulatedRecordBytes = 0; - closeAllQuietly(spillingChannel, spillFileReader, () -> spillFile.delete()); + closeAllQuietly(spillingChannel, spillFileReader, () -> spillFile.release()); spillingChannel = null; spillFileReader = null; spillFile = null; @@ -260,9 +260,10 @@ private FileChannel createSpillingChannel() throws IOException { int maxAttempts = 10; for (int attempt = 0; attempt < maxAttempts; attempt++) { String directory = tempDirs[rnd.nextInt(tempDirs.length)]; - spillFile = new File(directory, randomString(rnd) + ".inputchannel"); - if (spillFile.createNewFile()) { - return new RandomAccessFile(spillFile, "rw").getChannel(); + File file = new File(directory, randomString(rnd) + ".inputchannel"); + if (file.createNewFile()) { + spillFile = new RefCountedFile(file); + return new RandomAccessFile(file, "rw").getChannel(); } } From 2ed38c12be7151ab49e9cf2b4e2d8138f1ae4c62 Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Thu, 7 May 2020 16:48:47 +0200 Subject: [PATCH 058/773] [FLINK-17547][task] Implement getUnconsumedSegment for spilled buffers --- .../core/memory/MemorySegmentFactory.java | 28 ++++- .../apache/flink/util/CloseableIterator.java | 32 +++++ .../core/memory/MemorySegmentFactoryTest.java | 64 ++++++++++ .../flink/util/CloseableIteratorTest.java | 82 +++++++++++++ .../io/disk/FileBasedBufferIterator.java | 90 ++++++++++++++ .../api/serialization/SpanningWrapper.java | 54 +++++--- .../serialization/SpanningWrapperTest.java | 115 ++++++++++++++++++ 7 files changed, 448 insertions(+), 17 deletions(-) create mode 100644 flink-core/src/test/java/org/apache/flink/core/memory/MemorySegmentFactoryTest.java create mode 100644 flink-core/src/test/java/org/apache/flink/util/CloseableIteratorTest.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileBasedBufferIterator.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapperTest.java diff --git a/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegmentFactory.java b/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegmentFactory.java index ee301a1d7e708..f643bc4fb4da9 100644 --- a/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegmentFactory.java +++ b/flink-core/src/main/java/org/apache/flink/core/memory/MemorySegmentFactory.java @@ -27,6 +27,8 @@ import java.nio.ByteBuffer; +import static org.apache.flink.util.Preconditions.checkArgument; + /** * A factory for (hybrid) memory segments ({@link HybridMemorySegment}). * @@ -52,6 +54,31 @@ public static MemorySegment wrap(byte[] buffer) { return new HybridMemorySegment(buffer, null); } + /** + * Copies the given heap memory region and creates a new memory segment wrapping it. + * + * @param bytes The heap memory region. + * @param start starting position, inclusive + * @param end end position, exclusive + * @return A new memory segment that targets a copy of the given heap memory region. + * @throws IllegalArgumentException if start > end or end > bytes.length + */ + public static MemorySegment wrapCopy(byte[] bytes, int start, int end) throws IllegalArgumentException { + checkArgument(end >= start); + checkArgument(end <= bytes.length); + MemorySegment copy = allocateUnpooledSegment(end - start); + copy.put(0, bytes, start, copy.size()); + return copy; + } + + /** + * Wraps the four bytes representing the given number with a {@link MemorySegment}. + * @see ByteBuffer#putInt(int) + */ + public static MemorySegment wrapInt(int value) { + return wrap(ByteBuffer.allocate(Integer.BYTES).putInt(value).array()); + } + /** * Allocates some unpooled memory and creates a new memory segment that represents * that memory. @@ -161,5 +188,4 @@ public static MemorySegment allocateOffHeapUnsafeMemory(int size, Object owner, public static MemorySegment wrapOffHeapMemory(ByteBuffer memory) { return new HybridMemorySegment(memory, null); } - } diff --git a/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java b/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java index cc51324df93d7..e0c5ec05b19a8 100644 --- a/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java +++ b/flink-core/src/main/java/org/apache/flink/util/CloseableIterator.java @@ -24,7 +24,9 @@ import java.util.Collections; import java.util.Deque; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; +import java.util.Queue; import java.util.function.Consumer; import static java.util.Arrays.asList; @@ -80,6 +82,36 @@ public void close() throws Exception { }; } + static CloseableIterator flatten(CloseableIterator... iterators) { + return new CloseableIterator() { + private final Queue> queue = removeEmptyHead(new LinkedList<>(asList(iterators))); + + private Queue> removeEmptyHead(Queue> queue) { + while (!queue.isEmpty() && !queue.peek().hasNext()) { + queue.poll(); + } + return queue; + } + + @Override + public boolean hasNext() { + removeEmptyHead(queue); + return !queue.isEmpty(); + } + + @Override + public T next() { + removeEmptyHead(queue); + return queue.peek().next(); + } + + @Override + public void close() throws Exception { + IOUtils.closeAll(iterators); + } + }; + } + @SuppressWarnings("unchecked") static CloseableIterator empty() { return (CloseableIterator) EMPTY_INSTANCE; diff --git a/flink-core/src/test/java/org/apache/flink/core/memory/MemorySegmentFactoryTest.java b/flink-core/src/test/java/org/apache/flink/core/memory/MemorySegmentFactoryTest.java new file mode 100644 index 0000000000000..59c1d7ebd81ef --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/core/memory/MemorySegmentFactoryTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.core.memory; + +import org.junit.Test; + +import static java.lang.System.arraycopy; +import static org.junit.Assert.assertArrayEquals; + +/** + * {@link MemorySegmentFactory} test. + */ +public class MemorySegmentFactoryTest { + + @Test + public void testWrapCopyChangingData() { + byte[] data = {1, 2, 3, 4, 5}; + byte[] changingData = new byte[data.length]; + arraycopy(data, 0, changingData, 0, data.length); + MemorySegment segment = MemorySegmentFactory.wrapCopy(changingData, 0, changingData.length); + changingData[0]++; + assertArrayEquals(data, segment.heapMemory); + } + + @Test + public void testWrapPartialCopy() { + byte[] data = {1, 2, 3, 5, 6}; + MemorySegment segment = MemorySegmentFactory.wrapCopy(data, 0, data.length / 2); + byte[] exp = new byte[segment.size()]; + arraycopy(data, 0, exp, 0, exp.length); + assertArrayEquals(exp, segment.heapMemory); + } + + @Test + public void testWrapCopyEmpty() { + MemorySegmentFactory.wrapCopy(new byte[0], 0, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrapCopyWrongStart() { + MemorySegmentFactory.wrapCopy(new byte[]{1, 2, 3}, 10, 3); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrapCopyWrongEnd() { + MemorySegmentFactory.wrapCopy(new byte[]{1, 2, 3}, 0, 10); + } + +} diff --git a/flink-core/src/test/java/org/apache/flink/util/CloseableIteratorTest.java b/flink-core/src/test/java/org/apache/flink/util/CloseableIteratorTest.java new file mode 100644 index 0000000000000..e2d4d3f7b0733 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/util/CloseableIteratorTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.util; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; + +/** + * {@link CloseableIterator} test. + */ +@SuppressWarnings("unchecked") +public class CloseableIteratorTest { + + private static final String[] ELEMENTS = new String[]{"flink", "blink"}; + + @Test + public void testFlattenEmpty() throws Exception { + List> iterators = asList( + CloseableIterator.flatten(), + CloseableIterator.flatten(CloseableIterator.empty()), + CloseableIterator.flatten(CloseableIterator.flatten())); + for (CloseableIterator i : iterators) { + assertFalse(i.hasNext()); + i.close(); + } + } + + @Test + public void testFlattenIteration() { + CloseableIterator iterator = CloseableIterator.flatten( + CloseableIterator.ofElement(ELEMENTS[0], unused -> { + }), + CloseableIterator.ofElement(ELEMENTS[1], unused -> { + }) + ); + + List iterated = new ArrayList<>(); + iterator.forEachRemaining(iterated::add); + assertArrayEquals(ELEMENTS, iterated.toArray()); + } + + @Test(expected = TestException.class) + public void testFlattenErrorHandling() throws Exception { + List closed = new ArrayList<>(); + CloseableIterator iterator = CloseableIterator.flatten( + CloseableIterator.ofElement(ELEMENTS[0], e -> { + closed.add(e); + throw new TestException(); + }), + CloseableIterator.ofElement(ELEMENTS[1], closed::add) + ); + try { + iterator.close(); + } finally { + assertArrayEquals(ELEMENTS, closed.toArray()); + } + } + + private static class TestException extends RuntimeException { + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileBasedBufferIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileBasedBufferIterator.java new file mode 100644 index 0000000000000..c7e1cd8b975e4 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/FileBasedBufferIterator.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.disk; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.fs.RefCountedFile; +import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; +import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; +import org.apache.flink.util.CloseableIterator; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import static org.apache.flink.core.memory.MemorySegmentFactory.wrap; +import static org.apache.flink.runtime.io.network.buffer.Buffer.DataType.DATA_BUFFER; +import static org.apache.flink.util.IOUtils.closeAll; +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + +/** + * {@link CloseableIterator} of {@link Buffer buffers} over file content. + */ +@Internal +public class FileBasedBufferIterator implements CloseableIterator { + + private final RefCountedFile file; + private final FileInputStream stream; + private final int bufferSize; + + private int offset; + private int bytesToRead; + + public FileBasedBufferIterator(RefCountedFile file, int bytesToRead, int bufferSize) throws FileNotFoundException { + checkNotNull(file); + checkArgument(bytesToRead >= 0); + checkArgument(bufferSize > 0); + this.stream = new FileInputStream(file.getFile()); + this.file = file; + this.bufferSize = bufferSize; + this.bytesToRead = bytesToRead; + file.retain(); + } + + @Override + public boolean hasNext() { + return bytesToRead > 0; + } + + @Override + public Buffer next() { + byte[] buffer = new byte[bufferSize]; + int bytesRead = read(buffer); + checkState(bytesRead >= 0, "unexpected end of file, file = " + file.getFile() + ", offset=" + offset); + offset += bytesRead; + bytesToRead -= bytesRead; + return new NetworkBuffer(wrap(buffer), FreeingBufferRecycler.INSTANCE, DATA_BUFFER, bytesRead); + } + + private int read(byte[] buffer) { + int limit = Math.min(buffer.length, bytesToRead); + try { + return stream.read(buffer, offset, limit); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + closeAll(stream, file::release); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java index 9cffde37c10d7..45d6ad7ba4d3d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapper.java @@ -24,7 +24,10 @@ import org.apache.flink.core.memory.DataOutputSerializer; import org.apache.flink.core.memory.MemorySegment; import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.runtime.io.disk.FileBasedBufferIterator; import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler; +import org.apache.flink.runtime.io.network.buffer.NetworkBuffer; import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.StringUtils; @@ -41,15 +44,18 @@ import static java.lang.Math.max; import static java.lang.Math.min; +import static org.apache.flink.core.memory.MemorySegmentFactory.wrapCopy; +import static org.apache.flink.core.memory.MemorySegmentFactory.wrapInt; import static org.apache.flink.runtime.io.network.api.serialization.NonSpanningWrapper.singleBufferIterator; import static org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.LENGTH_BYTES; +import static org.apache.flink.util.CloseableIterator.empty; import static org.apache.flink.util.FileUtils.writeCompletely; import static org.apache.flink.util.IOUtils.closeAllQuietly; final class SpanningWrapper { - private static final int THRESHOLD_FOR_SPILLING = 5 * 1024 * 1024; // 5 MiBytes - private static final int FILE_BUFFER_SIZE = 2 * 1024 * 1024; + private static final int DEFAULT_THRESHOLD_FOR_SPILLING = 5 * 1024 * 1024; // 5 MiBytes + private static final int DEFAULT_FILE_BUFFER_SIZE = 2 * 1024 * 1024; private final byte[] initialBuffer = new byte[1024]; @@ -61,6 +67,8 @@ final class SpanningWrapper { final ByteBuffer lengthBuffer; + private final int fileBufferSize; + private FileChannel spillingChannel; private byte[] buffer; @@ -79,16 +87,21 @@ final class SpanningWrapper { private DataInputViewStreamWrapper spillFileReader; + private int thresholdForSpilling; + SpanningWrapper(String[] tempDirs) { - this.tempDirs = tempDirs; + this(tempDirs, DEFAULT_THRESHOLD_FOR_SPILLING, DEFAULT_FILE_BUFFER_SIZE); + } + SpanningWrapper(String[] tempDirectories, int threshold, int fileBufferSize) { + this.tempDirs = tempDirectories; this.lengthBuffer = ByteBuffer.allocate(LENGTH_BYTES); this.lengthBuffer.order(ByteOrder.BIG_ENDIAN); - this.recordLength = -1; - this.serializationReadBuffer = new DataInputDeserializer(); this.buffer = initialBuffer; + this.thresholdForSpilling = threshold; + this.fileBufferSize = fileBufferSize; } /** @@ -101,7 +114,7 @@ void transferFrom(NonSpanningWrapper partial, int nextRecordLength) throws IOExc } private boolean isAboveSpillingThreshold() { - return recordLength > THRESHOLD_FOR_SPILLING; + return recordLength > thresholdForSpilling; } void addNextChunkFromMemorySegment(MemorySegment segment, int offset, int numBytes) throws IOException { @@ -137,7 +150,7 @@ private void copyIntoFile(MemorySegment segment, int offset, int length) throws accumulatedRecordBytes += length; if (hasFullRecord()) { spillingChannel.close(); - spillFileReader = new DataInputViewStreamWrapper(new BufferedInputStream(new FileInputStream(spillFile.getFile()), FILE_BUFFER_SIZE)); + spillFileReader = new DataInputViewStreamWrapper(new BufferedInputStream(new FileInputStream(spillFile.getFile()), fileBufferSize)); } } @@ -170,22 +183,26 @@ private void updateLength(int length) throws IOException { CloseableIterator getUnconsumedSegment() throws IOException { if (isReadingLength()) { - return singleBufferIterator(copyLengthBuffer()); + return singleBufferIterator(wrapCopy(lengthBuffer.array(), 0, lengthBuffer.position())); } else if (isAboveSpillingThreshold()) { - throw new UnsupportedOperationException("Unaligned checkpoint currently do not support spilled records."); + return createSpilledDataIterator(); } else if (recordLength == -1) { - return CloseableIterator.empty(); // no remaining partial length or data + return empty(); // no remaining partial length or data } else { return singleBufferIterator(copyDataBuffer()); } } - private MemorySegment copyLengthBuffer() { - int position = lengthBuffer.position(); - MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(position); - lengthBuffer.position(0); - segment.put(0, lengthBuffer, position); - return segment; + @SuppressWarnings("unchecked") + private CloseableIterator createSpilledDataIterator() throws IOException { + if (spillingChannel != null && spillingChannel.isOpen()) { + spillingChannel.force(false); + } + return CloseableIterator.flatten( + toSingleBufferIterator(wrapInt(recordLength)), + new FileBasedBufferIterator(spillFile, min(accumulatedRecordBytes, recordLength), fileBufferSize), + leftOverData == null ? empty() : toSingleBufferIterator(wrapCopy(leftOverData.getArray(), leftOverStart, leftOverLimit)) + ); } private MemorySegment copyDataBuffer() throws IOException { @@ -289,4 +306,9 @@ private boolean isReadingLength() { return lengthBuffer.position() > 0; } + private static CloseableIterator toSingleBufferIterator(MemorySegment segment) { + NetworkBuffer buffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, Buffer.DataType.DATA_BUFFER, segment.size()); + return CloseableIterator.ofElement(buffer, Buffer::recycleBuffer); + } + } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapperTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapperTest.java new file mode 100644 index 0000000000000..be57e216135e9 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/SpanningWrapperTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.network.api.serialization; + +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.util.CloseableIterator; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.apache.flink.core.memory.MemorySegmentFactory.wrap; +import static org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer.LENGTH_BYTES; +import static org.junit.Assert.assertArrayEquals; + +/** + * {@link SpanningWrapper} test. + */ +public class SpanningWrapperTest { + + private static final Random random = new Random(); + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testLargeUnconsumedSegment() throws Exception { + int recordLen = 100; + int firstChunk = (int) (recordLen * .9); + int spillingThreshold = (int) (firstChunk * .9); + + byte[] record1 = recordBytes(recordLen); + byte[] record2 = recordBytes(recordLen * 2); + + SpanningWrapper spanningWrapper = new SpanningWrapper(new String[]{folder.newFolder().getAbsolutePath()}, spillingThreshold, recordLen); + spanningWrapper.transferFrom(wrapNonSpanning(record1, firstChunk), recordLen); + spanningWrapper.addNextChunkFromMemorySegment(wrap(record1), firstChunk, recordLen - firstChunk + LENGTH_BYTES); + spanningWrapper.addNextChunkFromMemorySegment(wrap(record2), 0, record2.length); + + CloseableIterator unconsumedSegment = spanningWrapper.getUnconsumedSegment(); + + spanningWrapper.getInputView().readFully(new byte[recordLen], 0, recordLen); // read out from file + spanningWrapper.transferLeftOverTo(new NonSpanningWrapper()); // clear any leftover + spanningWrapper.transferFrom(wrapNonSpanning(recordBytes(recordLen), recordLen), recordLen); // overwrite with new data + + assertArrayEquals(concat(record1, record2), toByteArray(unconsumedSegment)); + } + + private byte[] recordBytes(int recordLen) { + byte[] inputData = randomBytes(recordLen + LENGTH_BYTES); + for (int i = 0; i < Integer.BYTES; i++) { + inputData[Integer.BYTES - i - 1] = (byte) (recordLen >>> i * 8); + } + return inputData; + } + + private NonSpanningWrapper wrapNonSpanning(byte[] bytes, int len) { + NonSpanningWrapper nonSpanningWrapper = new NonSpanningWrapper(); + MemorySegment segment = wrap(bytes); + nonSpanningWrapper.initializeFromMemorySegment(segment, 0, len); + nonSpanningWrapper.readInt(); // emulate read length performed in getNextRecord to move position + return nonSpanningWrapper; + } + + private byte[] toByteArray(CloseableIterator unconsumed) { + final List buffers = new ArrayList<>(); + try { + unconsumed.forEachRemaining(buffers::add); + byte[] result = new byte[buffers.stream().mapToInt(Buffer::readableBytes).sum()]; + int offset = 0; + for (Buffer buffer : buffers) { + int len = buffer.readableBytes(); + buffer.getNioBuffer(0, len).get(result, offset, len); + offset += len; + } + return result; + } finally { + buffers.forEach(Buffer::recycleBuffer); + } + } + + private byte[] randomBytes(int length) { + byte[] inputData = new byte[length]; + random.nextBytes(inputData); + return inputData; + } + + private byte[] concat(byte[] input1, byte[] input2) { + byte[] expected = new byte[input1.length + input2.length]; + System.arraycopy(input1, 0, expected, 0, input1.length); + System.arraycopy(input2, 0, expected, input1.length, input2.length); + return expected; + } + +} From c34a4f288deb7dd349c6e52f674d7fa95aa013a1 Mon Sep 17 00:00:00 2001 From: Flavio Pompermaier Date: Mon, 4 May 2020 18:15:38 +0200 Subject: [PATCH 059/773] [FLINK-17361] Add custom query on JDBC tables --- docs/dev/table/connect.md | 6 +++- .../internal/options/JdbcReadOptions.java | 29 ++++++++++++++---- .../connector/jdbc/table/JdbcTableSource.java | 9 ++++-- .../table/JdbcTableSourceSinkFactory.java | 6 ++++ .../table/descriptors/JdbcValidator.java | 2 ++ .../jdbc/table/JdbcTableSourceITCase.java | 30 +++++++++++++++++++ .../table/JdbcTableSourceSinkFactoryTest.java | 2 ++ 7 files changed, 75 insertions(+), 9 deletions(-) diff --git a/docs/dev/table/connect.md b/docs/dev/table/connect.md index 4a1e83a394ab0..ac2646c771d90 100644 --- a/docs/dev/table/connect.md +++ b/docs/dev/table/connect.md @@ -1307,7 +1307,11 @@ CREATE TABLE MyUserTable ( 'connector.username' = 'name', 'connector.password' = 'password', - -- **followings are scan options, optional, used when reading from table** + -- **followings are scan options, optional, used when reading from a table** + + -- optional: SQL query / prepared statement. + -- If set, this will take precedence over the 'connector.table' setting + 'connector.read.query' = 'SELECT * FROM sometable', -- These options must all be specified if any of them is specified. In addition, -- partition.num must be specified. They describe how to partition the table when diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/internal/options/JdbcReadOptions.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/internal/options/JdbcReadOptions.java index a1350ab35eb75..65b57297c5f2b 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/internal/options/JdbcReadOptions.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/internal/options/JdbcReadOptions.java @@ -27,6 +27,7 @@ */ public class JdbcReadOptions implements Serializable { + private final String query; private final String partitionColumnName; private final Long partitionLowerBound; private final Long partitionUpperBound; @@ -35,11 +36,13 @@ public class JdbcReadOptions implements Serializable { private final int fetchSize; private JdbcReadOptions( + String query, String partitionColumnName, Long partitionLowerBound, Long partitionUpperBound, Integer numPartitions, int fetchSize) { + this.query = query; this.partitionColumnName = partitionColumnName; this.partitionLowerBound = partitionLowerBound; this.partitionUpperBound = partitionUpperBound; @@ -48,6 +51,10 @@ private JdbcReadOptions( this.fetchSize = fetchSize; } + public Optional getQuery() { + return Optional.ofNullable(query); + } + public Optional getPartitionColumnName() { return Optional.ofNullable(partitionColumnName); } @@ -76,11 +83,12 @@ public static Builder builder() { public boolean equals(Object o) { if (o instanceof JdbcReadOptions) { JdbcReadOptions options = (JdbcReadOptions) o; - return Objects.equals(partitionColumnName, options.partitionColumnName) && - Objects.equals(partitionLowerBound, options.partitionLowerBound) && - Objects.equals(partitionUpperBound, options.partitionUpperBound) && - Objects.equals(numPartitions, options.numPartitions) && - Objects.equals(fetchSize, options.fetchSize); + return Objects.equals(query, options.query) && + Objects.equals(partitionColumnName, options.partitionColumnName) && + Objects.equals(partitionLowerBound, options.partitionLowerBound) && + Objects.equals(partitionUpperBound, options.partitionUpperBound) && + Objects.equals(numPartitions, options.numPartitions) && + Objects.equals(fetchSize, options.fetchSize); } else { return false; } @@ -90,6 +98,7 @@ public boolean equals(Object o) { * Builder of {@link JdbcReadOptions}. */ public static class Builder { + protected String query; protected String partitionColumnName; protected Long partitionLowerBound; protected Long partitionUpperBound; @@ -97,6 +106,14 @@ public static class Builder { protected int fetchSize = 0; + /** + * optional, SQL query statement for this JDBC source. + */ + public Builder setQuery(String query) { + this.query = query; + return this; + } + /** * optional, name of the column used for partitioning the input. */ @@ -140,7 +157,7 @@ public Builder setFetchSize(int fetchSize) { public JdbcReadOptions build() { return new JdbcReadOptions( - partitionColumnName, partitionLowerBound, partitionUpperBound, numPartitions, fetchSize); + query, partitionColumnName, partitionLowerBound, partitionUpperBound, numPartitions, fetchSize); } } } diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSource.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSource.java index a599f2fb71a6f..ff21aae7223e4 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSource.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSource.java @@ -168,8 +168,7 @@ private JdbcInputFormat getInputFormat() { } final JdbcDialect dialect = options.getDialect(); - String query = dialect.getSelectFromStatement( - options.getTableName(), rowTypeInfo.getFieldNames(), new String[0]); + String query = getBaseQueryStatement(rowTypeInfo); if (readOptions.getPartitionColumnName().isPresent()) { long lowerBound = readOptions.getPartitionLowerBound().get(); long upperBound = readOptions.getPartitionUpperBound().get(); @@ -185,6 +184,12 @@ private JdbcInputFormat getInputFormat() { return builder.finish(); } + private String getBaseQueryStatement(RowTypeInfo rowTypeInfo) { + return readOptions.getQuery().orElseGet(() -> + options.getDialect().getSelectFromStatement( + options.getTableName(), rowTypeInfo.getFieldNames(), new String[0])); + } + @Override public boolean equals(Object o) { if (o instanceof JdbcTableSource) { diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactory.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactory.java index bdc8642a74343..438779f624cf2 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactory.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactory.java @@ -57,6 +57,7 @@ import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_READ_PARTITION_LOWER_BOUND; import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_READ_PARTITION_NUM; import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_READ_PARTITION_UPPER_BOUND; +import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_READ_QUERY; import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_TABLE; import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_TYPE_VALUE_JDBC; import static org.apache.flink.table.descriptors.JdbcValidator.CONNECTOR_URL; @@ -96,6 +97,7 @@ public List supportedProperties() { properties.add(CONNECTOR_PASSWORD); // scan options + properties.add(CONNECTOR_READ_QUERY); properties.add(CONNECTOR_READ_PARTITION_COLUMN); properties.add(CONNECTOR_READ_PARTITION_NUM); properties.add(CONNECTOR_READ_PARTITION_LOWER_BOUND); @@ -184,6 +186,7 @@ private JdbcOptions getJdbcOptions(DescriptorProperties descriptorProperties) { } private JdbcReadOptions getJdbcReadOptions(DescriptorProperties descriptorProperties) { + final Optional query = descriptorProperties.getOptionalString(CONNECTOR_READ_QUERY); final Optional partitionColumnName = descriptorProperties.getOptionalString(CONNECTOR_READ_PARTITION_COLUMN); final Optional partitionLower = descriptorProperties.getOptionalLong(CONNECTOR_READ_PARTITION_LOWER_BOUND); @@ -191,6 +194,9 @@ private JdbcReadOptions getJdbcReadOptions(DescriptorProperties descriptorProper final Optional numPartitions = descriptorProperties.getOptionalInt(CONNECTOR_READ_PARTITION_NUM); final JdbcReadOptions.Builder builder = JdbcReadOptions.builder(); + if (query.isPresent()) { + builder.setQuery(query.get()); + } if (partitionColumnName.isPresent()) { builder.setPartitionColumnName(partitionColumnName.get()); builder.setPartitionLowerBound(partitionLower.get()); diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/table/descriptors/JdbcValidator.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/table/descriptors/JdbcValidator.java index e8b0fe58aaca4..218759eb5fd0c 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/table/descriptors/JdbcValidator.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/table/descriptors/JdbcValidator.java @@ -43,6 +43,7 @@ public class JdbcValidator extends ConnectorDescriptorValidator { public static final String CONNECTOR_USERNAME = "connector.username"; public static final String CONNECTOR_PASSWORD = "connector.password"; + public static final String CONNECTOR_READ_QUERY = "connector.read.query"; public static final String CONNECTOR_READ_PARTITION_COLUMN = "connector.read.partition.column"; public static final String CONNECTOR_READ_PARTITION_LOWER_BOUND = "connector.read.partition.lower-bound"; public static final String CONNECTOR_READ_PARTITION_UPPER_BOUND = "connector.read.partition.upper-bound"; @@ -89,6 +90,7 @@ private void validateCommonProperties(DescriptorProperties properties) { } private void validateReadProperties(DescriptorProperties properties) { + properties.validateString(CONNECTOR_READ_QUERY, true); properties.validateString(CONNECTOR_READ_PARTITION_COLUMN, true); properties.validateLong(CONNECTOR_READ_PARTITION_LOWER_BOUND, true); properties.validateLong(CONNECTOR_READ_PARTITION_UPPER_BOUND, true); diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java index 74e90b2c22d14..fa8d98aaedd35 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java @@ -154,4 +154,34 @@ public void testProjectableJdbcSource() throws Exception { "2020-01-01T15:36:01.123456,101.1234"); StreamITCase.compareWithList(expected); } + + @Test + public void testScanQueryJDBCSource() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + EnvironmentSettings envSettings = EnvironmentSettings.newInstance() + .useBlinkPlanner() + .inStreamingMode() + .build(); + StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, envSettings); + + final String testQuery = "SELECT id FROM " + INPUT_TABLE; + tEnv.sqlUpdate( + "CREATE TABLE test(" + + "id BIGINT" + + ") WITH (" + + " 'connector.type'='jdbc'," + + " 'connector.url'='" + DB_URL + "'," + + " 'connector.table'='whatever'," + + " 'connector.read.query'='" + testQuery + "'" + + ")" + ); + + StreamITCase.clear(); + tEnv.toAppendStream(tEnv.sqlQuery("SELECT id FROM test"), Row.class) + .addSink(new StreamITCase.StringSink<>()); + env.execute(); + + List expected = Arrays.asList("1", "2"); + StreamITCase.compareWithList(expected); + } } diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactoryTest.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactoryTest.java index 38cb29c2d37be..7f15565edc081 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactoryTest.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceSinkFactoryTest.java @@ -88,6 +88,7 @@ public void testJdbcCommonProperties() { @Test public void testJdbcReadProperties() { Map properties = getBasicProperties(); + properties.put("connector.read.query", "SELECT aaa FROM mytable"); properties.put("connector.read.partition.column", "aaa"); properties.put("connector.read.partition.lower-bound", "-10"); properties.put("connector.read.partition.upper-bound", "100"); @@ -102,6 +103,7 @@ public void testJdbcReadProperties() { .setTableName("mytable") .build(); final JdbcReadOptions readOptions = JdbcReadOptions.builder() + .setQuery("SELECT aaa FROM mytable") .setPartitionColumnName("aaa") .setPartitionLowerBound(-10) .setPartitionUpperBound(100) From ecc464a51e87814201b650765e3971ef9ff0a2a1 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Tue, 19 May 2020 17:40:26 +0200 Subject: [PATCH 060/773] [FLINK-17361] Refactor JdbcTableSourceITCase to use TableResult instead of StreamITCase Using the static sink approach of StreamITCase is potentially problematic with concurrency, plus the code is just plain nicer like this. --- .../jdbc/table/JdbcTableSourceITCase.java | 85 +++++++++++-------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java index fa8d98aaedd35..277191c452ae8 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java @@ -21,8 +21,8 @@ import org.apache.flink.connector.jdbc.JdbcTestBase; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.runtime.utils.StreamITCase; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; @@ -34,8 +34,15 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.Assert.assertThat; /** @@ -107,20 +114,19 @@ public void testJdbcSource() throws Exception { ")" ); - StreamITCase.clear(); - tEnv.toAppendStream(tEnv.sqlQuery("SELECT * FROM " + INPUT_TABLE), Row.class) - .addSink(new StreamITCase.StringSink<>()); - env.execute(); + TableResult tableResult = tEnv.executeSql("SELECT * FROM " + INPUT_TABLE); + + List results = manifestResults(tableResult); - List expected = - Arrays.asList( - "1,2020-01-01T15:35:00.123456,2020-01-01T15:35:00.123456789,15:35,1.175E-37,1.79769E308,100.1234", - "2,2020-01-01T15:36:01.123456,2020-01-01T15:36:01.123456789,15:36:01,-1.175E-37,-1.79769E308,101.1234"); - StreamITCase.compareWithList(expected); + assertThat( + results, + containsInAnyOrder( + "1,2020-01-01T15:35:00.123456,2020-01-01T15:35:00.123456789,15:35,1.175E-37,1.79769E308,100.1234", + "2,2020-01-01T15:36:01.123456,2020-01-01T15:36:01.123456789,15:36:01,-1.175E-37,-1.79769E308,101.1234")); } @Test - public void testProjectableJdbcSource() throws Exception { + public void testProjectableJdbcSource() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); EnvironmentSettings envSettings = EnvironmentSettings.newInstance() .useBlinkPlanner() @@ -143,20 +149,19 @@ public void testProjectableJdbcSource() throws Exception { ")" ); - StreamITCase.clear(); - tEnv.toAppendStream(tEnv.sqlQuery("SELECT timestamp6_col, decimal_col FROM " + INPUT_TABLE), Row.class) - .addSink(new StreamITCase.StringSink<>()); - env.execute(); + TableResult tableResult = tEnv.executeSql("SELECT timestamp6_col, decimal_col FROM " + INPUT_TABLE); + + List results = manifestResults(tableResult); - List expected = - Arrays.asList( - "2020-01-01T15:35:00.123456,100.1234", - "2020-01-01T15:36:01.123456,101.1234"); - StreamITCase.compareWithList(expected); + assertThat( + results, + containsInAnyOrder( + "2020-01-01T15:35:00.123456,100.1234", + "2020-01-01T15:36:01.123456,101.1234")); } @Test - public void testScanQueryJDBCSource() throws Exception { + public void testScanQueryJDBCSource() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); EnvironmentSettings envSettings = EnvironmentSettings.newInstance() .useBlinkPlanner() @@ -165,23 +170,29 @@ public void testScanQueryJDBCSource() throws Exception { StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, envSettings); final String testQuery = "SELECT id FROM " + INPUT_TABLE; - tEnv.sqlUpdate( - "CREATE TABLE test(" + - "id BIGINT" + - ") WITH (" + - " 'connector.type'='jdbc'," + - " 'connector.url'='" + DB_URL + "'," + - " 'connector.table'='whatever'," + - " 'connector.read.query'='" + testQuery + "'" + - ")" + tEnv.executeSql( + "CREATE TABLE test(" + + "id BIGINT" + + ") WITH (" + + " 'connector.type'='jdbc'," + + " 'connector.url'='" + DB_URL + "'," + + " 'connector.table'='whatever'," + + " 'connector.read.query'='" + testQuery + "'" + + ")" ); - StreamITCase.clear(); - tEnv.toAppendStream(tEnv.sqlQuery("SELECT id FROM test"), Row.class) - .addSink(new StreamITCase.StringSink<>()); - env.execute(); + TableResult tableResult = tEnv.executeSql("SELECT id FROM test"); + + List results = manifestResults(tableResult); + + assertThat(results, containsInAnyOrder("1", "2")); + } - List expected = Arrays.asList("1", "2"); - StreamITCase.compareWithList(expected); + private static List manifestResults(TableResult result) { + Iterator resultIterator = result.collect(); + return StreamSupport + .stream(Spliterators.spliteratorUnknownSize(resultIterator, Spliterator.ORDERED), false) + .map(Row::toString) + .collect(Collectors.toList()); } } From ae5779e5e1352dfa67e57c5d34a468ca2cb1bcb9 Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Tue, 19 May 2020 20:00:51 +0800 Subject: [PATCH 061/773] [FLINK-17810][doc] Add document for K8s application mode This closes #12245 --- docs/ops/deployment/native_kubernetes.md | 44 +++++++++++++++++++-- docs/ops/deployment/native_kubernetes.zh.md | 44 +++++++++++++++++++-- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/docs/ops/deployment/native_kubernetes.md b/docs/ops/deployment/native_kubernetes.md index 00f783cebdff4..10bafa30a46f8 100644 --- a/docs/ops/deployment/native_kubernetes.md +++ b/docs/ops/deployment/native_kubernetes.md @@ -30,7 +30,7 @@ This page describes how to deploy a Flink session cluster natively on [Kubernete {:toc}

-Flink's native Kubernetes integration is still experimental. There may be changes in the configuration and CLI flags in latter versions. Job clusters are not yet supported. +Flink's native Kubernetes integration is still experimental. There may be changes in the configuration and CLI flags in latter versions.
## Requirements @@ -63,7 +63,7 @@ Although this setting may cause more cloud cost it has the effect that starting faster and during development you have more time to inspect the logfiles of your job. {% highlight bash %} -./bin/kubernetes-session.sh \ +$ ./bin/kubernetes-session.sh \ -Dkubernetes.cluster-id= \ -Dtaskmanager.memory.process.size=4096m \ -Dkubernetes.taskmanager.cpu=2 \ @@ -83,13 +83,13 @@ If you want to use a custom Docker image to deploy Flink containers, check [the If you created a custom Docker image you can provide it by setting the [`kubernetes.container.image`](../config.html#kubernetes-container-image) configuration option: {% highlight bash %} -./bin/kubernetes-session.sh \ +$ ./bin/kubernetes-session.sh \ -Dkubernetes.cluster-id= \ -Dtaskmanager.memory.process.size=4096m \ -Dkubernetes.taskmanager.cpu=2 \ -Dtaskmanager.numberOfTaskSlots=4 \ -Dresourcemanager.taskmanager-timeout=3600000 \ - -Dkubernetes.container.image= + -Dkubernetes.container.image= {% endhighlight %} ### Submitting jobs to an existing Session @@ -170,6 +170,42 @@ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m If the pod is running, you can use `kubectl exec -it bash` to tunnel in and view the logs or debug the process. +## Flink Kubernetes Application + +### Start Flink Application + +Application mode allows users to create a single image containing their Job and the Flink runtime, which will automatically create and destroy cluster components as needed. The Flink community provides base docker images [customized](docker.html#customize-flink-image) for any use case. + +{% highlight dockerfile %} +FROM flink +RUN mkdir -p $FLINK_HOME/usrlib +COPY /path/of/my-flink-job-*.jar $FLINK_HOME/usrlib/my-flink-job.jar +{% endhighlight %} + +Use the following command to start a Flink application. +{% highlight bash %} +$ ./bin/flink run-application -p 8 -t kubernetes-application \ + -Dkubernetes.cluster-id= \ + -Dtaskmanager.memory.process.size=4096m \ + -Dkubernetes.taskmanager.cpu=2 \ + -Dtaskmanager.numberOfTaskSlots=4 \ + -Dkubernetes.container.image= \ + local:///opt/flink/usrlib/my-flink-job.jar +{% endhighlight %} + +Note: Only "local" is supported as schema for application mode. This assumes that the jar is located in the image, not the Flink client. + +Note: All the jars in the "$FLINK_HOME/usrlib" directory in the image will be added to user classpath. + +### Stop Flink Application + +When an application is stopped, all Flink cluster resources are automatically destroyed. +As always, Jobs may stop when manually canceled or, in the case of bounded Jobs, complete. + +{% highlight bash %} +$ ./bin/flink cancel -t kubernetes-application -Dkubernetes.cluster-id= +{% endhighlight %} + ## Kubernetes concepts ### Namespaces diff --git a/docs/ops/deployment/native_kubernetes.zh.md b/docs/ops/deployment/native_kubernetes.zh.md index e341154af2d8d..e9e0506888d88 100644 --- a/docs/ops/deployment/native_kubernetes.zh.md +++ b/docs/ops/deployment/native_kubernetes.zh.md @@ -30,7 +30,7 @@ This page describes how to deploy a Flink session cluster natively on [Kubernete {:toc}
-Flink's native Kubernetes integration is still experimental. There may be changes in the configuration and CLI flags in latter versions. Job clusters are not yet supported. +Flink's native Kubernetes integration is still experimental. There may be changes in the configuration and CLI flags in latter versions.
## Requirements @@ -63,7 +63,7 @@ Although this setting may cause more cloud cost it has the effect that starting faster and during development you have more time to inspect the logfiles of your job. {% highlight bash %} -./bin/kubernetes-session.sh \ +$ ./bin/kubernetes-session.sh \ -Dkubernetes.cluster-id= \ -Dtaskmanager.memory.process.size=4096m \ -Dkubernetes.taskmanager.cpu=2 \ @@ -83,13 +83,13 @@ If you want to use a custom Docker image to deploy Flink containers, check [the If you created a custom Docker image you can provide it by setting the [`kubernetes.container.image`](../config.html#kubernetes-container-image) configuration option: {% highlight bash %} -./bin/kubernetes-session.sh \ +$ ./bin/kubernetes-session.sh \ -Dkubernetes.cluster-id= \ -Dtaskmanager.memory.process.size=4096m \ -Dkubernetes.taskmanager.cpu=2 \ -Dtaskmanager.numberOfTaskSlots=4 \ -Dresourcemanager.taskmanager-timeout=3600000 \ - -Dkubernetes.container.image= + -Dkubernetes.container.image= {% endhighlight %} ### Submitting jobs to an existing Session @@ -170,6 +170,42 @@ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m If the pod is running, you can use `kubectl exec -it bash` to tunnel in and view the logs or debug the process. +## Flink Kubernetes Application + +### Start Flink Application + +Application mode allows users to create a single image containing their Job and the Flink runtime, which will automatically create and destroy cluster components as needed. The Flink community provides base docker images [customized](docker.html#customize-flink-image) for any use case. + +{% highlight dockerfile %} +FROM flink +RUN mkdir -p $FLINK_HOME/usrlib +COPY /path/of/my-flink-job-*.jar $FLINK_HOME/usrlib/my-flink-job.jar +{% endhighlight %} + +Use the following command to start a Flink application. +{% highlight bash %} +$ ./bin/flink run-application -p 8 -t kubernetes-application \ + -Dkubernetes.cluster-id= \ + -Dtaskmanager.memory.process.size=4096m \ + -Dkubernetes.taskmanager.cpu=2 \ + -Dtaskmanager.numberOfTaskSlots=4 \ + -Dkubernetes.container.image= \ + local:///opt/flink/usrlib/my-flink-job.jar +{% endhighlight %} + +Note: Only "local" is supported as schema for application mode. This assumes that the jar is located in the image, not the Flink client. + +Note: All the jars in the "$FLINK_HOME/usrlib" directory in the image will be added to user classpath. + +### Stop Flink Application + +When an application is stopped, all Flink cluster resources are automatically destroyed. +As always, Jobs may stop when manually canceled or, in the case of bounded Jobs, complete. + +{% highlight bash %} +$ ./bin/flink cancel -t kubernetes-application -Dkubernetes.cluster-id= +{% endhighlight %} + ## Kubernetes concepts ### Namespaces From 98f578d1427b6b1a6d6ca71f500774835ad81047 Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Mon, 18 May 2020 17:40:17 +0800 Subject: [PATCH 062/773] [hotfix][table] Improve testing implementation for the new projection push down --- .../flink/table/utils/TableSchemaUtils.java | 20 ++ .../table/utils/TableSchemaUtilsTest.java | 48 +++ .../TestProjectableValuesTableFactory.java | 326 ------------------ .../factories/TestValuesTableFactory.java | 79 ++++- ...ushProjectIntoTableSourceScanRuleTest.java | 6 +- .../org.apache.flink.table.factories.Factory | 1 - .../planner/plan/stream/sql/TableScanTest.xml | 2 +- .../plan/batch/sql/TableSourceTest.scala | 4 +- .../plan/stream/sql/TableSourceTest.scala | 20 +- .../plan/stream/table/TableSourceTest.scala | 20 +- .../runtime/batch/sql/CalcITCase.scala | 11 +- .../runtime/stream/sql/CalcITCase.scala | 10 +- .../planner/runtime/utils/BatchTestBase.scala | 3 +- .../runtime/utils/StreamingTestBase.scala | 5 +- 14 files changed, 170 insertions(+), 385 deletions(-) delete mode 100644 flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestProjectableValuesTableFactory.java diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/TableSchemaUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/TableSchemaUtils.java index 4863cb2fa3262..67ee12548a042 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/TableSchemaUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/utils/TableSchemaUtils.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Optional; +import static org.apache.flink.util.Preconditions.checkArgument; + /** * Utilities to {@link TableSchema}. */ @@ -61,6 +63,24 @@ public static TableSchema getPhysicalSchema(TableSchema tableSchema) { return builder.build(); } + /** + * Creates a new {@link TableSchema} with the projected fields from another {@link TableSchema}. + * The new {@link TableSchema} doesn't contain any primary key or watermark information. + * + * @see org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown + */ + public static TableSchema projectSchema(TableSchema tableSchema, int[][] projectedFields) { + checkArgument(!containsGeneratedColumns(tableSchema), "It's illegal to project on a schema contains computed columns."); + TableSchema.Builder schemaBuilder = TableSchema.builder(); + List tableColumns = tableSchema.getTableColumns(); + for (int[] fieldPath : projectedFields) { + checkArgument(fieldPath.length == 1, "Nested projection push down is not supported yet."); + TableColumn column = tableColumns.get(fieldPath[0]); + schemaBuilder.field(column.getName(), column.getType()); + } + return schemaBuilder.build(); + } + /** * Returns true if there are any generated columns in the given {@link TableColumn}. */ diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/utils/TableSchemaUtilsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/utils/TableSchemaUtilsTest.java index e96ddd94df69a..3e760b1b38df3 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/utils/TableSchemaUtilsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/utils/TableSchemaUtilsTest.java @@ -70,4 +70,52 @@ public void testDropConstraint() { exceptionRule.expectMessage("Constraint ct2 to drop does not exist"); TableSchemaUtils.dropConstraint(oriSchema, "ct2"); } + + @Test + public void testInvalidProjectSchema() { + { + TableSchema schema = TableSchema.builder() + .field("a", DataTypes.INT().notNull()) + .field("b", DataTypes.STRING()) + .field("c", DataTypes.INT(), "a + 1") + .field("t", DataTypes.TIMESTAMP(3)) + .primaryKey("ct1", new String[]{"a"}) + .watermark("t", "t", DataTypes.TIMESTAMP(3)) + .build(); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("It's illegal to project on a schema contains computed columns."); + int[][] projectedFields = {{1}}; + TableSchemaUtils.projectSchema(schema, projectedFields); + } + + { + TableSchema schema = TableSchema.builder() + .field("a", DataTypes.ROW(DataTypes.FIELD("f0", DataTypes.STRING()))) + .field("b", DataTypes.STRING()) + .build(); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("Nested projection push down is not supported yet."); + int[][] projectedFields = {{0, 1}}; + TableSchemaUtils.projectSchema(schema, projectedFields); + } + } + + @Test + public void testProjectSchema() { + TableSchema schema = TableSchema.builder() + .field("a", DataTypes.INT().notNull()) + .field("b", DataTypes.STRING()) + .field("t", DataTypes.TIMESTAMP(3)) + .primaryKey("a") + .watermark("t", "t", DataTypes.TIMESTAMP(3)) + .build(); + + int[][] projectedFields = {{2}, {0}}; + TableSchema projected = TableSchemaUtils.projectSchema(schema, projectedFields); + TableSchema expected = TableSchema.builder() + .field("t", DataTypes.TIMESTAMP(3)) + .field("a", DataTypes.INT().notNull()) + .build(); + assertEquals(expected, projected); + } } diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestProjectableValuesTableFactory.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestProjectableValuesTableFactory.java deleted file mode 100644 index c5367ea1c8e84..0000000000000 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestProjectableValuesTableFactory.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.planner.factories; - -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.java.io.CollectionInputFormat; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.configuration.ConfigOptions; -import org.apache.flink.streaming.api.functions.source.FromElementsFunction; -import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.connector.source.DynamicTableSource; -import org.apache.flink.table.connector.source.InputFormatProvider; -import org.apache.flink.table.connector.source.ScanTableSource; -import org.apache.flink.table.connector.source.SourceFunctionProvider; -import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; -import org.apache.flink.table.data.RowData; -import org.apache.flink.table.factories.DynamicTableSourceFactory; -import org.apache.flink.table.factories.FactoryUtil; -import org.apache.flink.table.planner.utils.JavaScalaConversionUtil; -import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.FieldsDataType; -import org.apache.flink.table.types.logical.RowType; -import org.apache.flink.types.Row; -import org.apache.flink.types.RowKind; -import org.apache.flink.util.Preconditions; - -import javax.annotation.Nullable; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import scala.collection.Seq; - -/** - * Test implementation of {@link DynamicTableSourceFactory} that supports projection push down. - */ -public class TestProjectableValuesTableFactory implements DynamicTableSourceFactory { - - // -------------------------------------------------------------------------------------------- - // Data Registration - // -------------------------------------------------------------------------------------------- - - private static final AtomicInteger idCounter = new AtomicInteger(0); - private static final Map>> registeredData = new HashMap<>(); - - /** - * Register the given data into the data factory context and return the data id. - * The data id can be used as a reference to the registered data in data connector DDL. - */ - public static String registerData(Collection data) { - List> dataWithKinds = new ArrayList<>(); - for (Row row : data) { - dataWithKinds.add(Tuple2.of(RowKind.INSERT, row)); - } - return registerChangelogData(dataWithKinds); - } - - /** - * Register the given data into the data factory context and return the data id. - * The data id can be used as a reference to the registered data in data connector DDL. - */ - public static String registerData(Seq data) { - return registerData(JavaScalaConversionUtil.toJava(data)); - } - - /** - * Register the given data with RowKind into the data factory context and return the data id. - * The data id can be used as a reference to the registered data in data connector DDL. - * TODO: remove this utility once Row supports RowKind. - */ - public static String registerChangelogData(Collection> data) { - String id = String.valueOf(idCounter.incrementAndGet()); - registeredData.put(id, data); - return id; - } - - /** - * Removes the registered data under the given data id. - */ - public static void clearAllRegisteredData() { - registeredData.clear(); - } - - // -------------------------------------------------------------------------------------------- - // Factory - // -------------------------------------------------------------------------------------------- - - private static final String IDENTIFIER = "projectable-values"; - - private static final ConfigOption DATA_ID = ConfigOptions - .key("data-id") - .stringType() - .defaultValue(null); - - private static final ConfigOption BOUNDED = ConfigOptions - .key("bounded") - .booleanType() - .defaultValue(false); - - private static final ConfigOption CHANGELOG_MODE = ConfigOptions - .key("changelog-mode") - .stringType() - .defaultValue("I"); // all available "I,UA,UB,D" - - private static final ConfigOption RUNTIME_SOURCE = ConfigOptions - .key("runtime-source") - .stringType() - .defaultValue("SourceFunction"); // another is "InputFormat" - - private static final ConfigOption NESTED_PROJECTION_SUPPORTED = ConfigOptions - .key("nested-projection-supported") - .booleanType() - .defaultValue(false); - - @Override - public String factoryIdentifier() { - return IDENTIFIER; - } - - @Override - public DynamicTableSource createDynamicTableSource(Context context) { - FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); - helper.validate(); - ChangelogMode changelogMode = parseChangelogMode(helper.getOptions().get(CHANGELOG_MODE)); - String runtimeSource = helper.getOptions().get(RUNTIME_SOURCE); - boolean isBounded = helper.getOptions().get(BOUNDED); - String dataId = helper.getOptions().get(DATA_ID); - boolean nestedProjectionSupported = helper.getOptions().get(NESTED_PROJECTION_SUPPORTED); - - Collection> data = registeredData.getOrDefault(dataId, Collections.emptyList()); - DataType rowDataType = context.getCatalogTable().getSchema().toPhysicalRowDataType(); - return new TestProjectableValuesTableSource( - changelogMode, - isBounded, - runtimeSource, - rowDataType, - data, - nestedProjectionSupported); - } - - @Override - public Set> requiredOptions() { - return Collections.emptySet(); - } - - @Override - public Set> optionalOptions() { - return new HashSet<>(Arrays.asList( - DATA_ID, - CHANGELOG_MODE, - BOUNDED, - RUNTIME_SOURCE, - NESTED_PROJECTION_SUPPORTED)); - } - - private ChangelogMode parseChangelogMode(String string) { - ChangelogMode.Builder builder = ChangelogMode.newBuilder(); - for (String split : string.split(",")) { - switch (split.trim()) { - case "I": - builder.addContainedKind(RowKind.INSERT); - break; - case "UB": - builder.addContainedKind(RowKind.UPDATE_BEFORE); - break; - case "UA": - builder.addContainedKind(RowKind.UPDATE_AFTER); - break; - case "D": - builder.addContainedKind(RowKind.DELETE); - break; - default: - throw new IllegalArgumentException("Invalid ChangelogMode string: " + string); - } - } - return builder.build(); - } - - // -------------------------------------------------------------------------------------------- - // Table source - // -------------------------------------------------------------------------------------------- - - /** - * Values {@link DynamicTableSource} for testing. - */ - private static class TestProjectableValuesTableSource implements ScanTableSource, SupportsProjectionPushDown { - - private final ChangelogMode changelogMode; - private final boolean bounded; - private final String runtimeSource; - private DataType physicalRowDataType; - private final Collection> data; - private final boolean nestedProjectionSupported; - private int[] projectedFields = null; - - private TestProjectableValuesTableSource( - ChangelogMode changelogMode, - boolean bounded, String runtimeSource, - DataType physicalRowDataType, - Collection> data, - boolean nestedProjectionSupported) { - this.changelogMode = changelogMode; - this.bounded = bounded; - this.runtimeSource = runtimeSource; - this.physicalRowDataType = physicalRowDataType; - this.data = data; - this.nestedProjectionSupported = nestedProjectionSupported; - } - - @Override - public ChangelogMode getChangelogMode() { - return changelogMode; - } - - @SuppressWarnings("unchecked") - @Override - public ScanRuntimeProvider getScanRuntimeProvider(ScanTableSource.Context runtimeProviderContext) { - TypeSerializer serializer = (TypeSerializer) runtimeProviderContext - .createTypeInformation(physicalRowDataType) - .createSerializer(new ExecutionConfig()); - DataStructureConverter converter = runtimeProviderContext.createDataStructureConverter(physicalRowDataType); - Collection values = convertToRowData(data, projectedFields, converter); - - if (runtimeSource.equals("SourceFunction")) { - try { - return SourceFunctionProvider.of( - new FromElementsFunction<>(serializer, values), - bounded); - } catch (IOException e) { - throw new RuntimeException(e); - } - } else if (runtimeSource.equals("InputFormat")) { - return InputFormatProvider.of(new CollectionInputFormat<>(values, serializer)); - } else { - throw new IllegalArgumentException("Unsupported runtime source class: " + runtimeSource); - } - } - - @Override - public DynamicTableSource copy() { - TestProjectableValuesTableSource newTableSource = new TestProjectableValuesTableSource( - changelogMode, bounded, runtimeSource, physicalRowDataType, data, nestedProjectionSupported); - newTableSource.projectedFields = projectedFields; - return newTableSource; - } - - @Override - public String asSummaryString() { - return "TestProjectableValues"; - } - - private static Collection convertToRowData( - Collection> data, - @Nullable int[] projectedFields, - DataStructureConverter converter) { - List result = new ArrayList<>(); - for (Tuple2 value : data) { - Row projectedRow; - if (projectedFields == null) { - projectedRow = value.f1; - } else { - Object[] newValues = new Object[projectedFields.length]; - for (int i = 0; i < projectedFields.length; ++i) { - newValues[i] = value.f1.getField(projectedFields[i]); - } - projectedRow = Row.of(newValues); - } - RowData rowData = (RowData) converter.toInternal(projectedRow); - if (rowData != null) { - rowData.setRowKind(value.f0); - result.add(rowData); - } - } - return result; - } - - @Override - public boolean supportsNestedProjection() { - return nestedProjectionSupported; - } - - @Override - public void applyProjection(int[][] projectedFields) { - this.projectedFields = new int[projectedFields.length]; - FieldsDataType dataType = (FieldsDataType) physicalRowDataType; - RowType rowType = ((RowType) physicalRowDataType.getLogicalType()); - DataTypes.Field[] fields = new DataTypes.Field[projectedFields.length]; - for (int i = 0; i < projectedFields.length; ++i) { - int[] projection = projectedFields[i]; - Preconditions.checkArgument(projection.length == 1); - int index = projection[0]; - this.projectedFields[i] = index; - fields[i] = DataTypes.FIELD(rowType.getFieldNames().get(index), dataType.getChildren().get(index)); - } - this.physicalRowDataType = DataTypes.ROW(fields); - } - } -} diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java index e889814a45b8e..47dcf428e18f6 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java @@ -40,6 +40,7 @@ import org.apache.flink.table.connector.source.SourceFunctionProvider; import org.apache.flink.table.connector.source.TableFunctionProvider; import org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown; +import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; import org.apache.flink.table.data.RowData; import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.factories.DynamicTableSinkFactory; @@ -55,7 +56,6 @@ import org.apache.flink.table.planner.factories.TestValuesRuntimeFunctions.TestValuesLookupFunction; import org.apache.flink.table.planner.utils.JavaScalaConversionUtil; import org.apache.flink.table.runtime.typeutils.RowDataTypeInfo; -import org.apache.flink.table.types.DataType; import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; @@ -235,6 +235,11 @@ private static RowKind parseRowKind(String rowKindShortString) { .booleanType() .defaultValue(true); + private static final ConfigOption NESTED_PROJECTION_SUPPORTED = ConfigOptions + .key("nested-projection-supported") + .booleanType() + .defaultValue(false); + @Override public String factoryIdentifier() { return IDENTIFIER; @@ -251,18 +256,21 @@ public DynamicTableSource createDynamicTableSource(Context context) { String sourceClass = helper.getOptions().get(TABLE_SOURCE_CLASS); boolean isAsync = helper.getOptions().get(ASYNC_ENABLED); String lookupFunctionClass = helper.getOptions().get(LOOKUP_FUNCTION_CLASS); + boolean nestedProjectionSupported = helper.getOptions().get(NESTED_PROJECTION_SUPPORTED); if (sourceClass.equals("DEFAULT")) { Collection> data = registeredData.getOrDefault(dataId, Collections.emptyList()); - DataType rowDataType = context.getCatalogTable().getSchema().toPhysicalRowDataType(); + TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); return new TestValuesTableSource( + physicalSchema, changelogMode, isBounded, runtimeSource, - rowDataType, data, isAsync, - lookupFunctionClass); + lookupFunctionClass, + nestedProjectionSupported, + null); } else { try { return InstantiationUtil.instantiate( @@ -306,7 +314,8 @@ public Set> optionalOptions() { ASYNC_ENABLED, TABLE_SOURCE_CLASS, SINK_INSERT_ONLY, - RUNTIME_SINK)); + RUNTIME_SINK, + NESTED_PROJECTION_SUPPORTED)); } private ChangelogMode parseChangelogMode(String string) { @@ -339,30 +348,37 @@ private ChangelogMode parseChangelogMode(String string) { /** * Values {@link DynamicTableSource} for testing. */ - private static class TestValuesTableSource implements ScanTableSource, LookupTableSource { + private static class TestValuesTableSource implements ScanTableSource, LookupTableSource, SupportsProjectionPushDown { + private TableSchema physicalSchema; private final ChangelogMode changelogMode; private final boolean bounded; private final String runtimeSource; - private final DataType physicalRowDataType; private final Collection> data; private final boolean isAsync; private final @Nullable String lookupFunctionClass; + private final boolean nestedProjectionSupported; + private @Nullable int[] projectedFields; private TestValuesTableSource( + TableSchema physicalSchema, ChangelogMode changelogMode, - boolean bounded, String runtimeSource, - DataType physicalRowDataType, + boolean bounded, + String runtimeSource, Collection> data, boolean isAsync, - @Nullable String lookupFunctionClass) { + @Nullable String lookupFunctionClass, + boolean nestedProjectionSupported, + int[] projectedFields) { + this.physicalSchema = physicalSchema; this.changelogMode = changelogMode; this.bounded = bounded; this.runtimeSource = runtimeSource; - this.physicalRowDataType = physicalRowDataType; this.data = data; this.isAsync = isAsync; this.lookupFunctionClass = lookupFunctionClass; + this.nestedProjectionSupported = nestedProjectionSupported; + this.projectedFields = projectedFields; } @Override @@ -374,11 +390,11 @@ public ChangelogMode getChangelogMode() { @Override public ScanRuntimeProvider getScanRuntimeProvider(ScanTableSource.Context runtimeProviderContext) { TypeSerializer serializer = (TypeSerializer) runtimeProviderContext - .createTypeInformation(physicalRowDataType) + .createTypeInformation(physicalSchema.toRowDataType()) .createSerializer(new ExecutionConfig()); - DataStructureConverter converter = runtimeProviderContext.createDataStructureConverter(physicalRowDataType); + DataStructureConverter converter = runtimeProviderContext.createDataStructureConverter(physicalSchema.toRowDataType()); converter.open(RuntimeConverter.Context.create(TestValuesTableFactory.class.getClassLoader())); - Collection values = convertToRowData(data, converter); + Collection values = convertToRowData(data, projectedFields, converter); if (runtimeSource.equals("SourceFunction")) { try { @@ -437,9 +453,29 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupTableSource.Context } } + @Override + public boolean supportsNestedProjection() { + return nestedProjectionSupported; + } + + @Override + public void applyProjection(int[][] projectedFields) { + this.physicalSchema = TableSchemaUtils.projectSchema(physicalSchema, projectedFields); + this.projectedFields = Arrays.stream(projectedFields).mapToInt(f -> f[0]).toArray(); + } + @Override public DynamicTableSource copy() { - return new TestValuesTableSource(changelogMode, bounded, runtimeSource, physicalRowDataType, data, isAsync, lookupFunctionClass); + return new TestValuesTableSource( + physicalSchema, + changelogMode, + bounded, + runtimeSource, + data, + isAsync, + lookupFunctionClass, + nestedProjectionSupported, + projectedFields); } @Override @@ -449,10 +485,21 @@ public String asSummaryString() { private static Collection convertToRowData( Collection> data, + int[] projectedFields, DataStructureConverter converter) { List result = new ArrayList<>(); for (Tuple2 value : data) { - RowData rowData = (RowData) converter.toInternal(value.f1); + Row projectedRow; + if (projectedFields == null) { + projectedRow = value.f1; + } else { + Object[] newValues = new Object[projectedFields.length]; + for (int i = 0; i < projectedFields.length; ++i) { + newValues[i] = value.f1.getField(projectedFields[i]); + } + projectedRow = Row.of(newValues); + } + RowData rowData = (RowData) converter.toInternal(projectedRow); if (rowData != null) { rowData.setRowKind(value.f0); result.add(rowData); diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java index 955e3d4495580..b8a5b0b43e2ed 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRuleTest.java @@ -54,7 +54,7 @@ public void setup() { " b bigint,\n" + " c string\n" + ") WITH (\n" + - " 'connector' = 'projectable-values',\n" + + " 'connector' = 'values',\n" + " 'bounded' = 'true'\n" + ")"; util().tableEnv().executeSql(ddl1); @@ -66,7 +66,7 @@ public void setup() { " c string,\n" + " d as a + 1\n" + ") WITH (\n" + - " 'connector' = 'projectable-values',\n" + + " 'connector' = 'values',\n" + " 'bounded' = 'true'\n" + ")"; util().tableEnv().executeSql(ddl2); @@ -92,7 +92,7 @@ private void testNestedProject(boolean nestedProjectionSupported) { " nested row,\n" + " name string\n" + ") WITH (\n" + - " 'connector' = 'projectable-values',\n" + + " 'connector' = 'values',\n" + " 'nested-projection-supported' = '" + nestedProjectionSupported + "',\n" + " 'bounded' = 'true'\n" + ")"; diff --git a/flink-table/flink-table-planner-blink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-planner-blink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory index 7632f4b906e4f..498fb982fd820 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-table/flink-table-planner-blink/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -14,5 +14,4 @@ # limitations under the License. org.apache.flink.table.planner.factories.TestValuesTableFactory -org.apache.flink.table.planner.factories.TestProjectableValuesTableFactory org.apache.flink.table.planner.utils.TestCsvFileSystemFormatFactory diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.xml index 9c08f740005bb..5e8c3fe274d60 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.xml @@ -55,7 +55,7 @@ LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) GroupAggregate(select=[COUNT_RETRACT(*) AS EXPR$0], changelogMode=[I,UA,D]) +- Exchange(distribution=[single], changelogMode=[I,UB,UA]) +- Calc(select=[0 AS $f0], where=[>(a, 1)], changelogMode=[I,UB,UA]) - +- TableSourceScan(table=[[default_catalog, default_database, src]], fields=[ts, a, b], changelogMode=[I,UB,UA]) + +- TableSourceScan(table=[[default_catalog, default_database, src, project=[a]]], fields=[a], changelogMode=[I,UB,UA]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSourceTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSourceTest.scala index 7b12fcfa045e2..9699a0580c289 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSourceTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSourceTest.scala @@ -35,7 +35,7 @@ class TableSourceTest extends TableTestBase { | b bigint, | c varchar(32) |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'true' |) """.stripMargin @@ -49,7 +49,7 @@ class TableSourceTest extends TableTestBase { | nested row, | name string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'true' |) |""".stripMargin diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.scala index 5f1568083f274..b5f252b997cce 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.scala @@ -37,7 +37,7 @@ class TableSourceTest extends TableTestBase { | name varchar(32), | watermark for rowtime as rowtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -57,7 +57,7 @@ class TableSourceTest extends TableTestBase { | name varchar(32), | watermark for rowtime as rowtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -86,7 +86,7 @@ class TableSourceTest extends TableTestBase { | pTime as PROCTIME(), | watermark for pTime as pTime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -107,7 +107,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for ptime as ptime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -128,7 +128,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for rtime as rtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -148,7 +148,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for rtime as rtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -168,7 +168,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for ptime as ptime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -188,7 +188,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for rtime as rtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -208,7 +208,7 @@ class TableSourceTest extends TableTestBase { | nested row, | name string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'nested-projection-supported' = 'false', | 'bounded' = 'false' |) @@ -235,7 +235,7 @@ class TableSourceTest extends TableTestBase { | id int, | name varchar(32) |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala index 9d3ce0512d227..0f0ead91fcfc7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala @@ -39,7 +39,7 @@ class TableSourceTest extends TableTestBase { | name varchar(32), | watermark for rowtime as rowtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -61,7 +61,7 @@ class TableSourceTest extends TableTestBase { | name varchar(32), | watermark for rowtime as rowtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -86,7 +86,7 @@ class TableSourceTest extends TableTestBase { | proctime as PROCTIME(), | watermark for proctime as proctime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -107,7 +107,7 @@ class TableSourceTest extends TableTestBase { | name varchar(32), | proctime as PROCTIME() |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -132,7 +132,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for ptime as ptime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -154,7 +154,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for rtime as rtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -175,7 +175,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for rtime as rtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -196,7 +196,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for ptime as ptime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -217,7 +217,7 @@ class TableSourceTest extends TableTestBase { | ptime as PROCTIME(), | watermark for rtime as rtime |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'bounded' = 'false' |) """.stripMargin @@ -238,7 +238,7 @@ class TableSourceTest extends TableTestBase { | nested row, | name string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'nested-projection-supported' = 'false', | 'bounded' = 'false' |) diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CalcITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CalcITCase.scala index dda3bbee5bd88..2d799d1fecf14 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CalcITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CalcITCase.scala @@ -31,7 +31,7 @@ import org.apache.flink.table.api.config.ExecutionConfigOptions import org.apache.flink.table.data.{DecimalDataUtils, TimestampData} import org.apache.flink.table.data.util.DataFormatConverters.LocalDateConverter import org.apache.flink.table.planner.expressions.utils.{RichFunc1, RichFunc2, RichFunc3, SplitUDF} -import org.apache.flink.table.planner.factories.TestProjectableValuesTableFactory +import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.plan.rules.physical.batch.BatchExecSortRule import org.apache.flink.table.planner.runtime.utils.BatchTableEnvUtil.parseFieldNames import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row @@ -42,7 +42,6 @@ import org.apache.flink.table.planner.utils.DateTimeTestUtil import org.apache.flink.table.planner.utils.DateTimeTestUtil._ import org.apache.flink.table.runtime.functions.SqlDateTimeUtils.unixTimestampToLocalDateTime import org.apache.flink.types.Row - import org.junit.Assert.assertEquals import org.junit._ @@ -1250,7 +1249,7 @@ class CalcITCase extends BatchTestBase { @Test def testSimpleProject(): Unit = { - val myTableDataId = TestProjectableValuesTableFactory.registerData(TestData.smallData3) + val myTableDataId = TestValuesTableFactory.registerData(TestData.smallData3) val ddl = s""" |CREATE TABLE SimpleTable ( @@ -1258,7 +1257,7 @@ class CalcITCase extends BatchTestBase { | b bigint, | c string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'data-id' = '$myTableDataId', | 'bounded' = 'true' |) @@ -1278,7 +1277,7 @@ class CalcITCase extends BatchTestBase { row(2, row(row("HELLO", 22), row(222, false)), row("hello", 2222), "mary"), row(3, row(row("HELLO WORLD", 33), row(333, true)), row("hello world", 3333), "benji") ) - val myTableDataId = TestProjectableValuesTableFactory.registerData(data) + val myTableDataId = TestValuesTableFactory.registerData(data) val ddl = s""" |CREATE TABLE NestedTable ( @@ -1288,7 +1287,7 @@ class CalcITCase extends BatchTestBase { | nested row, | name string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'nested-projection-supported' = 'false', | 'data-id' = '$myTableDataId', | 'bounded' = 'true' diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala index 7cc6e6673ef0f..d32c9ea05e155 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala @@ -25,7 +25,7 @@ import org.apache.flink.api.scala.typeutils.Types import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.api.scala._ import org.apache.flink.table.data.{GenericRowData, RowData} -import org.apache.flink.table.planner.factories.TestProjectableValuesTableFactory +import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.table.runtime.typeutils.RowDataTypeInfo @@ -290,7 +290,7 @@ class CalcITCase extends StreamingTestBase { @Test def testSimpleProject(): Unit = { - val myTableDataId = TestProjectableValuesTableFactory.registerData(TestData.smallData3) + val myTableDataId = TestValuesTableFactory.registerData(TestData.smallData3) val ddl = s""" |CREATE TABLE SimpleTable ( @@ -298,7 +298,7 @@ class CalcITCase extends StreamingTestBase { | b bigint, | c string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'data-id' = '$myTableDataId', | 'bounded' = 'true' |) @@ -321,7 +321,7 @@ class CalcITCase extends StreamingTestBase { row(2, row(row("HELLO", 22), row(222, false)), row("hello", 2222), "mary"), row(3, row(row("HELLO WORLD", 33), row(333, true)), row("hello world", 3333), "benji") ) - val myTableDataId = TestProjectableValuesTableFactory.registerData(data) + val myTableDataId = TestValuesTableFactory.registerData(data) val ddl = s""" |CREATE TABLE NestedTable ( @@ -331,7 +331,7 @@ class CalcITCase extends StreamingTestBase { | nested row, | name string |) WITH ( - | 'connector' = 'projectable-values', + | 'connector' = 'values', | 'nested-projection-supported' = 'false', | 'data-id' = '$myTableDataId', | 'bounded' = 'true' diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala index 5b59ffb6b8a68..403a24002a381 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/BatchTestBase.scala @@ -32,7 +32,7 @@ import org.apache.flink.table.data.binary.BinaryRowData import org.apache.flink.table.data.writer.BinaryRowWriter import org.apache.flink.table.functions.{AggregateFunction, ScalarFunction, TableFunction} import org.apache.flink.table.planner.delegation.PlannerBase -import org.apache.flink.table.planner.factories.{TestProjectableValuesTableFactory, TestValuesTableFactory} +import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil import org.apache.flink.table.planner.runtime.utils.BatchAbstractTestBase.DEFAULT_PARALLELISM @@ -81,7 +81,6 @@ class BatchTestBase extends BatchAbstractTestBase { @After def after(): Unit = { TestValuesTableFactory.clearAllData() - TestProjectableValuesTableFactory.clearAllRegisteredData() } /** diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala index 3b6bbe41fc017..3fffbc0f15c7e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala @@ -22,8 +22,8 @@ import org.apache.flink.api.common.JobExecutionResult import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.scala.StreamTableEnvironment -import org.apache.flink.table.api.{EnvironmentSettings, ImplicitExpressionConversions} -import org.apache.flink.table.planner.factories.{TestProjectableValuesTableFactory, TestValuesTableFactory} +import org.apache.flink.table.api.ImplicitExpressionConversions +import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.api.{EnvironmentSettings, Table} import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.types.Row @@ -62,7 +62,6 @@ class StreamingTestBase extends AbstractTestBase { def after(): Unit = { StreamTestSink.clear() TestValuesTableFactory.clearAllData() - TestProjectableValuesTableFactory.clearAllRegisteredData() } /** From 8fc79e674e4596be16db264b517025c26ccefcb3 Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Mon, 18 May 2020 17:48:08 +0800 Subject: [PATCH 063/773] [FLINK-17797][connector/hbase] Align the behavior between the new and legacy HBase table source This closes #12221 --- .../flink-connector-hbase/pom.xml | 8 ++ .../hbase/HBaseDynamicTableFactory.java | 43 +++++- .../hbase/source/HBaseDynamicTableSource.java | 21 ++- .../connector/hbase/util/HBaseSerde.java | 24 ++-- .../hbase/util/HBaseTableSchema.java | 12 -- .../connector/hbase/HBaseTablePlanTest.java | 127 ++++++++++++++++++ .../connector/hbase/HBaseTablePlanTest.xml | 36 +++++ 7 files changed, 242 insertions(+), 29 deletions(-) create mode 100644 flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseTablePlanTest.java create mode 100644 flink-connectors/flink-connector-hbase/src/test/resources/org/apache/flink/connector/hbase/HBaseTablePlanTest.xml diff --git a/flink-connectors/flink-connector-hbase/pom.xml b/flink-connectors/flink-connector-hbase/pom.xml index 90b16dbd10e1e..8e7a5a1d62daf 100644 --- a/flink-connectors/flink-connector-hbase/pom.xml +++ b/flink-connectors/flink-connector-hbase/pom.xml @@ -205,6 +205,14 @@ under the License. + + org.apache.flink + flink-core + ${project.version} + test + test-jar + + org.apache.flink flink-clients_${scala.binary.version} diff --git a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/HBaseDynamicTableFactory.java b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/HBaseDynamicTableFactory.java index ba855772a0535..64b381f455601 100644 --- a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/HBaseDynamicTableFactory.java +++ b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/HBaseDynamicTableFactory.java @@ -103,15 +103,15 @@ public class HBaseDynamicTableFactory implements DynamicTableSourceFactory, Dyna public DynamicTableSource createDynamicTableSource(Context context) { TableFactoryHelper helper = createTableFactoryHelper(this, context); helper.validate(); + TableSchema tableSchema = context.getCatalogTable().getSchema(); + validatePrimaryKey(tableSchema); + String hTableName = helper.getOptions().get(TABLE_NAME); // create default configuration from current runtime env (`hbase-site.xml` in classpath) first, Configuration hbaseClientConf = HBaseConfiguration.create(); hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, helper.getOptions().get(ZOOKEEPER_QUORUM)); hbaseClientConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, helper.getOptions().get(ZOOKEEPER_ZNODE_PARENT)); - String nullStringLiteral = helper.getOptions().get(NULL_STRING_LITERAL); - - TableSchema tableSchema = context.getCatalogTable().getSchema(); HBaseTableSchema hbaseSchema = HBaseTableSchema.fromTableSchema(tableSchema); return new HBaseDynamicTableSource( @@ -125,6 +125,9 @@ public DynamicTableSource createDynamicTableSource(Context context) { public DynamicTableSink createDynamicTableSink(Context context) { TableFactoryHelper helper = createTableFactoryHelper(this, context); helper.validate(); + TableSchema tableSchema = context.getCatalogTable().getSchema(); + validatePrimaryKey(tableSchema); + HBaseOptions.Builder hbaseOptionsBuilder = HBaseOptions.builder(); hbaseOptionsBuilder.setTableName(helper.getOptions().get(TABLE_NAME)); hbaseOptionsBuilder.setZkQuorum(helper.getOptions().get(ZOOKEEPER_QUORUM)); @@ -136,10 +139,7 @@ public DynamicTableSink createDynamicTableSink(Context context) { .ifPresent(v -> writeBuilder.setBufferFlushIntervalMillis(v.toMillis())); helper.getOptions().getOptional(SINK_BUFFER_FLUSH_MAX_ROWS) .ifPresent(writeBuilder::setBufferFlushMaxRows); - String nullStringLiteral = helper.getOptions().get(NULL_STRING_LITERAL); - - TableSchema tableSchema = context.getCatalogTable().getSchema(); HBaseTableSchema hbaseSchema = HBaseTableSchema.fromTableSchema(tableSchema); return new HBaseDynamicTableSink( @@ -172,4 +172,35 @@ public Set> optionalOptions() { set.add(SINK_BUFFER_FLUSH_INTERVAL); return set; } + + // ------------------------------------------------------------------------------------------ + + /** + * Checks that the HBase table have row key defined. A row key is defined as an atomic type, + * and column families and qualifiers are defined as ROW type. There shouldn't be multiple + * atomic type columns in the schema. The PRIMARY KEY constraint is optional, if exist, the + * primary key constraint must be defined on the single row key field. + */ + private static void validatePrimaryKey(TableSchema schema) { + HBaseTableSchema hbaseSchema = HBaseTableSchema.fromTableSchema(schema); + if (!hbaseSchema.getRowKeyName().isPresent()) { + throw new IllegalArgumentException( + "HBase table requires to define a row key field. " + + "A row key field is defined as an atomic type, " + + "column families and qualifiers are defined as ROW type."); + } + schema.getPrimaryKey().ifPresent(k -> { + if (k.getColumns().size() > 1) { + throw new IllegalArgumentException( + "HBase table doesn't support a primary Key on multiple columns. " + + "The primary key of HBase table must be defined on row key field."); + } + if (!hbaseSchema.getRowKeyName().get().equals(k.getColumns().get(0))) { + throw new IllegalArgumentException( + "Primary key of HBase table must be defined on the row key field. " + + "A row key field is defined as an atomic type, " + + "column families and qualifiers are defined as ROW type."); + } + }); + } } diff --git a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/source/HBaseDynamicTableSource.java b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/source/HBaseDynamicTableSource.java index e59a12e5aba17..dcc5a5b1b7e5f 100644 --- a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/source/HBaseDynamicTableSource.java +++ b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/source/HBaseDynamicTableSource.java @@ -21,12 +21,15 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.connector.hbase.util.HBaseTableSchema; +import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.connector.source.InputFormatProvider; import org.apache.flink.table.connector.source.LookupTableSource; import org.apache.flink.table.connector.source.ScanTableSource; import org.apache.flink.table.connector.source.TableFunctionProvider; +import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; +import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.hadoop.conf.Configuration; @@ -36,11 +39,11 @@ * HBase table source implementation. */ @Internal -public class HBaseDynamicTableSource implements ScanTableSource, LookupTableSource { +public class HBaseDynamicTableSource implements ScanTableSource, LookupTableSource, SupportsProjectionPushDown { private final Configuration conf; private final String tableName; - private final HBaseTableSchema hbaseSchema; + private HBaseTableSchema hbaseSchema; private final String nullStringLiteral; public HBaseDynamicTableSource( @@ -77,6 +80,20 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupTableSource.Context return TableFunctionProvider.of(new HBaseLookupFunction(conf, tableName, hbaseSchema)); } + @Override + public boolean supportsNestedProjection() { + // planner doesn't support nested projection push down yet. + return false; + } + + @Override + public void applyProjection(int[][] projectedFields) { + TableSchema projectSchema = TableSchemaUtils.projectSchema( + hbaseSchema.convertsToTableSchema(), + projectedFields); + this.hbaseSchema = HBaseTableSchema.fromTableSchema(projectSchema); + } + @Override public ChangelogMode getChangelogMode() { return ChangelogMode.insertOnly(); diff --git a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseSerde.java b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseSerde.java index ed4a11f6b7488..e5a377fcffcb9 100644 --- a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseSerde.java +++ b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseSerde.java @@ -67,8 +67,8 @@ public class HBaseSerde { private GenericRowData reusedRow; private GenericRowData[] reusedFamilyRows; - private final FieldEncoder keyEncoder; - private final FieldDecoder keyDecoder; + private final @Nullable FieldEncoder keyEncoder; + private final @Nullable FieldDecoder keyDecoder; private final FieldEncoder[][] qualifierEncoders; private final FieldDecoder[][] qualifierDecoders; @@ -78,18 +78,21 @@ public HBaseSerde(HBaseTableSchema hbaseSchema, final String nullStringLiteral) LogicalType rowkeyType = hbaseSchema.getRowKeyDataType().map(DataType::getLogicalType).orElse(null); // field length need take row key into account if it exists. - checkArgument(rowkeyIndex != -1 && rowkeyType != null, "row key is not set."); - this.fieldLength = families.length + 1; + if (rowkeyIndex != -1 && rowkeyType != null) { + this.fieldLength = families.length + 1; + this.keyEncoder = createFieldEncoder(rowkeyType); + this.keyDecoder = createFieldDecoder(rowkeyType); + } else { + this.fieldLength = families.length; + this.keyEncoder = null; + this.keyDecoder = null; + } this.nullStringBytes = nullStringLiteral.getBytes(StandardCharsets.UTF_8); // prepare output rows this.reusedRow = new GenericRowData(fieldLength); this.reusedFamilyRows = new GenericRowData[families.length]; - // row key should never be null - this.keyEncoder = createFieldEncoder(rowkeyType); - this.keyDecoder = createFieldDecoder(rowkeyType); - this.qualifiers = new byte[families.length][][]; this.qualifierEncoders = new FieldEncoder[families.length][]; this.qualifierDecoders = new FieldDecoder[families.length][]; @@ -115,6 +118,7 @@ public HBaseSerde(HBaseTableSchema hbaseSchema, final String nullStringLiteral) * @return The appropriate instance of Put for this use case. */ public @Nullable Put createPutMutation(RowData row) { + checkArgument(keyEncoder != null, "row key is not set."); byte[] rowkey = keyEncoder.encode(row, rowkeyIndex); if (rowkey.length == 0) { // drop dirty records, rowkey shouldn't be zero length @@ -146,6 +150,7 @@ public HBaseSerde(HBaseTableSchema hbaseSchema, final String nullStringLiteral) * @return The appropriate instance of Delete for this use case. */ public @Nullable Delete createDeleteMutation(RowData row) { + checkArgument(keyEncoder != null, "row key is not set."); byte[] rowkey = keyEncoder.encode(row, rowkeyIndex); if (rowkey.length == 0) { // drop dirty records, rowkey shouldn't be zero length @@ -189,9 +194,10 @@ public Scan createScan() { * Converts HBase {@link Result} into {@link RowData}. */ public RowData convertToRow(Result result) { - Object rowkey = keyDecoder.decode(result.getRow()); for (int i = 0; i < fieldLength; i++) { if (rowkeyIndex == i) { + assert keyDecoder != null; + Object rowkey = keyDecoder.decode(result.getRow()); reusedRow.setField(rowkeyIndex, rowkey); } else { int f = (rowkeyIndex != -1 && i > rowkeyIndex) ? i - 1 : i; diff --git a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseTableSchema.java b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseTableSchema.java index 41108f4bbbc82..116b1ae374fd0 100644 --- a/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseTableSchema.java +++ b/flink-connectors/flink-connector-hbase/src/main/java/org/apache/flink/connector/hbase/util/HBaseTableSchema.java @@ -361,18 +361,6 @@ public static HBaseTableSchema fromTableSchema(TableSchema schema) { "Unsupported field type '" + fieldType + "' for HBase."); } } - schema.getPrimaryKey().ifPresent(k -> { - if (k.getColumns().size() > 1 || - !hbaseSchema.getRowKeyName().isPresent() || - !hbaseSchema.getRowKeyName().get().equals(k.getColumns().get(0))) { - throw new IllegalArgumentException( - "Primary Key of HBase table should only be defined on the row key field."); - } - }); - if (!hbaseSchema.getRowKeyName().isPresent()) { - throw new IllegalArgumentException( - "HBase table requires to define a row key field. A row key field must be an atomic type."); - } return hbaseSchema; } diff --git a/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseTablePlanTest.java b/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseTablePlanTest.java new file mode 100644 index 0000000000000..053cf9982413e --- /dev/null +++ b/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseTablePlanTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.hbase; + +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.planner.utils.StreamTableTestUtil; +import org.apache.flink.table.planner.utils.TableTestBase; + +import org.junit.Test; + +import static org.apache.flink.util.CoreMatchers.containsCause; + +/** + * Plan tests for HBase connector, for example, testing projection push down. + */ +public class HBaseTablePlanTest extends TableTestBase { + + private final StreamTableTestUtil util = streamTestUtil(new TableConfig()); + + @Test + public void testMultipleRowKey() { + util.tableEnv().executeSql( + "CREATE TABLE hTable (" + + " family1 ROW," + + " family2 ROW," + + " rowkey INT," + + " rowkey2 STRING " + + ") WITH (" + + " 'connector' = 'hbase-1.4'," + + " 'table-name' = 'my_table'," + + " 'zookeeper.quorum' = 'localhost:2021'" + + ")"); + thrown().expect(containsCause(new IllegalArgumentException("Row key can't be set multiple times."))); + util.verifyPlan("SELECT * FROM hTable"); + } + + @Test + public void testNoneRowKey() { + util.tableEnv().executeSql( + "CREATE TABLE hTable (" + + " family1 ROW," + + " family2 ROW" + + ") WITH (" + + " 'connector' = 'hbase-1.4'," + + " 'table-name' = 'my_table'," + + " 'zookeeper.quorum' = 'localhost:2021'" + + ")"); + thrown().expect(containsCause(new IllegalArgumentException( + "HBase table requires to define a row key field. " + + "A row key field is defined as an atomic type, " + + "column families and qualifiers are defined as ROW type."))); + util.verifyPlan("SELECT * FROM hTable"); + } + + @Test + public void testInvalidPrimaryKey() { + util.tableEnv().executeSql( + "CREATE TABLE hTable (" + + " family1 ROW," + + " family2 ROW," + + " rowkey STRING, " + + " PRIMARY KEY (family1) NOT ENFORCED " + + ") WITH (" + + " 'connector' = 'hbase-1.4'," + + " 'table-name' = 'my_table'," + + " 'zookeeper.quorum' = 'localhost:2021'" + + ")"); + thrown().expect(containsCause(new IllegalArgumentException( + "Primary key of HBase table must be defined on the row key field. " + + "A row key field is defined as an atomic type, " + + "column families and qualifiers are defined as ROW type."))); + util.verifyPlan("SELECT * FROM hTable"); + } + + @Test + public void testUnsupportedDataType() { + util.tableEnv().executeSql( + "CREATE TABLE hTable (" + + " family1 ROW," + + " family2 ROW," + + " col1 ARRAY, " + + " rowkey STRING, " + + " PRIMARY KEY (rowkey) NOT ENFORCED " + + ") WITH (" + + " 'connector' = 'hbase-1.4'," + + " 'table-name' = 'my_table'," + + " 'zookeeper.quorum' = 'localhost:2021'" + + ")"); + thrown().expect(containsCause(new IllegalArgumentException( + "Unsupported field type 'ARRAY' for HBase."))); + util.verifyPlan("SELECT * FROM hTable"); + } + + @Test + public void testProjectionPushDown() { + util.tableEnv().executeSql( + "CREATE TABLE hTable (" + + " family1 ROW," + + " family2 ROW," + + " family3 ROW," + + " rowkey INT," + + " PRIMARY KEY (rowkey) NOT ENFORCED" + + ") WITH (" + + " 'connector' = 'hbase-1.4'," + + " 'table-name' = 'my_table'," + + " 'zookeeper.quorum' = 'localhost:2021'" + + ")"); + util.verifyPlan("SELECT h.family3, h.family2.col2 FROM hTable AS h"); + } + +} diff --git a/flink-connectors/flink-connector-hbase/src/test/resources/org/apache/flink/connector/hbase/HBaseTablePlanTest.xml b/flink-connectors/flink-connector-hbase/src/test/resources/org/apache/flink/connector/hbase/HBaseTablePlanTest.xml new file mode 100644 index 0000000000000..8391b1bc1a80e --- /dev/null +++ b/flink-connectors/flink-connector-hbase/src/test/resources/org/apache/flink/connector/hbase/HBaseTablePlanTest.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + From b5bcb22f28ace028f824cef4512aaf90ec18b69a Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Mon, 18 May 2020 17:48:25 +0800 Subject: [PATCH 064/773] [FLINK-17798][connector/jdbc] Align the behavior between the new and legacy JDBC table source This closes #12221 --- flink-connectors/flink-connector-jdbc/pom.xml | 2 +- .../jdbc/table/JdbcDynamicTableSource.java | 48 +++++++++-------- .../JdbcDynamicTableSourceSinkFactory.java | 7 +-- .../table/JdbcDynamicTableSourceITCase.java | 45 +++++++++------- ...ITCase.java => JdbcLookupTableITCase.java} | 43 ++++++++------- .../jdbc/table/JdbcTablePlanTest.java | 54 +++++++++++++++++++ .../jdbc/table/JdbcTableSourceITCase.java | 1 - .../jdbc/table/JdbcTablePlanTest.xml | 35 ++++++++++++ 8 files changed, 168 insertions(+), 67 deletions(-) rename flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/{JdbcLookupFunctionITCase.java => JdbcLookupTableITCase.java} (86%) create mode 100644 flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.java create mode 100644 flink-connectors/flink-connector-jdbc/src/test/resources/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.xml diff --git a/flink-connectors/flink-connector-jdbc/pom.xml b/flink-connectors/flink-connector-jdbc/pom.xml index a9595465feec2..29467e2bb83aa 100644 --- a/flink-connectors/flink-connector-jdbc/pom.xml +++ b/flink-connectors/flink-connector-jdbc/pom.xml @@ -94,7 +94,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner-blink_${scala.binary.version} ${project.version} test-jar test diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSource.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSource.java index 248ffe1b10b62..21a80a26ab748 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSource.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSource.java @@ -32,37 +32,35 @@ import org.apache.flink.table.connector.source.LookupTableSource; import org.apache.flink.table.connector.source.ScanTableSource; import org.apache.flink.table.connector.source.TableFunctionProvider; +import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.flink.util.Preconditions; -import java.util.Arrays; import java.util.Objects; /** * A {@link DynamicTableSource} for JDBC. */ @Internal -public class JdbcDynamicTableSource implements ScanTableSource, LookupTableSource { +public class JdbcDynamicTableSource implements ScanTableSource, LookupTableSource, SupportsProjectionPushDown { private final JdbcOptions options; private final JdbcReadOptions readOptions; private final JdbcLookupOptions lookupOptions; - private final TableSchema schema; - private final int[] selectFields; + private TableSchema physicalSchema; private final String dialectName; public JdbcDynamicTableSource( JdbcOptions options, JdbcReadOptions readOptions, JdbcLookupOptions lookupOptions, - TableSchema schema, - int[] selectFields) { + TableSchema physicalSchema) { this.options = options; this.readOptions = readOptions; this.lookupOptions = lookupOptions; - this.schema = schema; - this.selectFields = selectFields; + this.physicalSchema = physicalSchema; this.dialectName = options.getDialect().dialectName(); } @@ -74,15 +72,15 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupTableSource.Context int[] innerKeyArr = context.getKeys()[i]; Preconditions.checkArgument(innerKeyArr.length == 1, "JDBC only support non-nested look up keys"); - keyNames[i] = schema.getFieldNames()[innerKeyArr[0]]; + keyNames[i] = physicalSchema.getFieldNames()[innerKeyArr[0]]; } - final RowType rowType = (RowType) schema.toRowDataType().getLogicalType(); + final RowType rowType = (RowType) physicalSchema.toRowDataType().getLogicalType(); return TableFunctionProvider.of(new JdbcRowDataLookupFunction( options, lookupOptions, - schema.getFieldNames(), - schema.getFieldDataTypes(), + physicalSchema.getFieldNames(), + physicalSchema.getFieldDataTypes(), keyNames, rowType)); } @@ -101,7 +99,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanTableSource.Context runtim } final JdbcDialect dialect = options.getDialect(); String query = dialect.getSelectFromStatement( - options.getTableName(), schema.getFieldNames(), new String[0]); + options.getTableName(), physicalSchema.getFieldNames(), new String[0]); if (readOptions.getPartitionColumnName().isPresent()) { long lowerBound = readOptions.getPartitionLowerBound().get(); long upperBound = readOptions.getPartitionUpperBound().get(); @@ -113,10 +111,10 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanTableSource.Context runtim " BETWEEN ? AND ?"; } builder.setQuery(query); - final RowType rowType = (RowType) schema.toRowDataType().getLogicalType(); + final RowType rowType = (RowType) physicalSchema.toRowDataType().getLogicalType(); builder.setRowConverter(dialect.getRowConverter(rowType)); builder.setRowDataTypeInfo((TypeInformation) runtimeProviderContext - .createTypeInformation(schema.toRowDataType())); + .createTypeInformation(physicalSchema.toRowDataType())); return InputFormatProvider.of(builder.build()); } @@ -126,9 +124,20 @@ public ChangelogMode getChangelogMode() { return ChangelogMode.insertOnly(); } + @Override + public boolean supportsNestedProjection() { + // JDBC doesn't support nested projection + return false; + } + + @Override + public void applyProjection(int[][] projectedFields) { + this.physicalSchema = TableSchemaUtils.projectSchema(physicalSchema, projectedFields); + } + @Override public DynamicTableSource copy() { - return new JdbcDynamicTableSource(options, readOptions, lookupOptions, schema, selectFields); + return new JdbcDynamicTableSource(options, readOptions, lookupOptions, physicalSchema); } @Override @@ -148,15 +157,12 @@ public boolean equals(Object o) { return Objects.equals(options, that.options) && Objects.equals(readOptions, that.readOptions) && Objects.equals(lookupOptions, that.lookupOptions) && - Objects.equals(schema, that.schema) && - Arrays.equals(selectFields, that.selectFields) && + Objects.equals(physicalSchema, that.physicalSchema) && Objects.equals(dialectName, that.dialectName); } @Override public int hashCode() { - int result = Objects.hash(options, readOptions, lookupOptions, schema, dialectName); - result = 31 * result + Arrays.hashCode(selectFields); - return result; + return Objects.hash(options, readOptions, lookupOptions, physicalSchema, dialectName); } } diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceSinkFactory.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceSinkFactory.java index 28a129df2d641..930a1b012cf2f 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceSinkFactory.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceSinkFactory.java @@ -173,16 +173,11 @@ public DynamicTableSource createDynamicTableSource(Context context) { helper.validate(); validateConfigOptions(config); TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); - int[] selectFields = new int[physicalSchema.getFieldNames().length]; - for (int i = 0; i < selectFields.length; i++) { - selectFields[i] = i; - } return new JdbcDynamicTableSource( getJdbcOptions(helper.getOptions()), getJdbcReadOptions(helper.getOptions()), getJdbcLookupOptions(helper.getOptions()), - physicalSchema, - selectFields); + physicalSchema); } private JdbcOptions getJdbcOptions(ReadableConfig readableConfig) { diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java index 6f93307aaac0d..48be89e6c0b5a 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java @@ -22,10 +22,12 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.runtime.utils.StreamITCase; +import org.apache.flink.table.planner.runtime.utils.StreamTestSink; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -34,8 +36,12 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; /** * ITCase for {@link JdbcDynamicTableSource}. @@ -79,6 +85,7 @@ public void clearOutputTable() throws Exception { Statement stat = conn.createStatement()) { stat.executeUpdate("DROP TABLE " + INPUT_TABLE); } + StreamTestSink.clear(); } @Test @@ -106,16 +113,17 @@ public void testJdbcSource() throws Exception { ")" ); - StreamITCase.clear(); - tEnv.toAppendStream(tEnv.sqlQuery("SELECT * FROM " + INPUT_TABLE), Row.class) - .addSink(new StreamITCase.StringSink<>()); - env.execute(); - + Iterator collected = tEnv.executeSql("SELECT * FROM " + INPUT_TABLE).collect(); + List result = Lists.newArrayList(collected).stream() + .map(Row::toString) + .sorted() + .collect(Collectors.toList()); List expected = - Arrays.asList( + Stream.of( "1,2020-01-01T15:35:00.123456,2020-01-01T15:35:00.123456789,15:35,1.175E-37,1.79769E308,100.1234", - "2,2020-01-01T15:36:01.123456,2020-01-01T15:36:01.123456789,15:36:01,-1.175E-37,-1.79769E308,101.1234"); - StreamITCase.compareWithList(expected); + "2,2020-01-01T15:36:01.123456,2020-01-01T15:36:01.123456789,15:36:01,-1.175E-37,-1.79769E308,101.1234") + .sorted().collect(Collectors.toList()); + assertEquals(expected, result); } @Test @@ -147,15 +155,16 @@ public void testProject() throws Exception { ")" ); - StreamITCase.clear(); - tEnv.toAppendStream(tEnv.sqlQuery("SELECT id,timestamp6_col,decimal_col FROM " + INPUT_TABLE), Row.class) - .addSink(new StreamITCase.StringSink<>()); - env.execute(); - + Iterator collected = tEnv.executeSql("SELECT id,timestamp6_col,decimal_col FROM " + INPUT_TABLE).collect(); + List result = Lists.newArrayList(collected).stream() + .map(Row::toString) + .sorted() + .collect(Collectors.toList()); List expected = - Arrays.asList( + Stream.of( "1,2020-01-01T15:35:00.123456,100.1234", - "2,2020-01-01T15:36:01.123456,101.1234"); - StreamITCase.compareWithList(expected); + "2,2020-01-01T15:36:01.123456,101.1234") + .sorted().collect(Collectors.toList()); + assertEquals(expected, result); } } diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupFunctionITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java similarity index 86% rename from flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupFunctionITCase.java rename to flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java index 8d40cdd8889e0..793ea9d8f1234 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupFunctionITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java @@ -22,17 +22,17 @@ import org.apache.flink.connector.jdbc.JdbcTestFixture; import org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions; import org.apache.flink.connector.jdbc.internal.options.JdbcOptions; -import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.runtime.utils.StreamITCase; import org.apache.flink.table.types.DataType; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; + import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -46,16 +46,20 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; import static org.apache.flink.connector.jdbc.JdbcTestFixture.DERBY_EBOOKSHOP_DB; import static org.apache.flink.table.api.Expressions.$; +import static org.junit.Assert.assertEquals; /** - * IT case for {@link JdbcLookupFunction}. + * IT case for lookup source of JDBC connector. */ @RunWith(Parameterized.class) -public class JdbcLookupFunctionITCase extends AbstractTestBase { +public class JdbcLookupTableITCase extends AbstractTestBase { public static final String DB_URL = "jdbc:derby:memory:lookup"; public static final String LOOKUP_TABLE = "lookup_table"; @@ -63,7 +67,7 @@ public class JdbcLookupFunctionITCase extends AbstractTestBase { private final String tableFactory; private final boolean useCache; - public JdbcLookupFunctionITCase(String tableFactory, boolean useCache) { + public JdbcLookupTableITCase(String tableFactory, boolean useCache) { this.useCache = useCache; this.tableFactory = tableFactory; } @@ -143,16 +147,20 @@ public void clearOutputTable() throws Exception { } @Test - public void test() throws Exception { + public void testLookup() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); - StreamITCase.clear(); + Iterator collected; if ("legacyFactory".equals(tableFactory)) { - useLegacyTableFactory(env, tEnv); + collected = useLegacyTableFactory(env, tEnv); } else { - useDynamicTableFactory(env, tEnv); + collected = useDynamicTableFactory(env, tEnv); } + List result = Lists.newArrayList(collected).stream() + .map(Row::toString) + .sorted() + .collect(Collectors.toList()); List expected = new ArrayList<>(); expected.add("1,1,11-c1-v1,11-c2-v1"); @@ -162,11 +170,12 @@ public void test() throws Exception { expected.add("2,3,null,23-c2"); expected.add("2,5,25-c1,25-c2"); expected.add("3,8,38-c1,38-c2"); + Collections.sort(expected); - StreamITCase.compareWithList(expected); + assertEquals(expected, result); } - private void useLegacyTableFactory(StreamExecutionEnvironment env, StreamTableEnvironment tEnv) throws Exception { + private Iterator useLegacyTableFactory(StreamExecutionEnvironment env, StreamTableEnvironment tEnv) throws Exception { Table t = tEnv.fromDataStream(env.fromCollection(Arrays.asList( new Tuple2<>(1, "1"), new Tuple2<>(1, "1"), @@ -195,13 +204,10 @@ private void useLegacyTableFactory(StreamExecutionEnvironment env, StreamTableEn String sqlQuery = "SELECT id1, id2, comment1, comment2 FROM T, " + "LATERAL TABLE(jdbcLookup(id1, id2)) AS S(l_id1, l_id2, comment1, comment2)"; - Table result = tEnv.sqlQuery(sqlQuery); - DataStream resultSet = tEnv.toAppendStream(result, Row.class); - resultSet.addSink(new StreamITCase.StringSink<>()); - env.execute(); + return tEnv.executeSql(sqlQuery).collect(); } - private void useDynamicTableFactory(StreamExecutionEnvironment env, StreamTableEnvironment tEnv) throws Exception { + private Iterator useDynamicTableFactory(StreamExecutionEnvironment env, StreamTableEnvironment tEnv) throws Exception { Table t = tEnv.fromDataStream(env.fromCollection(Arrays.asList( new Tuple2<>(1, "1"), new Tuple2<>(1, "1"), @@ -229,9 +235,6 @@ private void useDynamicTableFactory(StreamExecutionEnvironment env, StreamTableE String sqlQuery = "SELECT source.id1, source.id2, L.comment1, L.comment2 FROM T AS source " + "JOIN lookup for system_time as of source.proctime AS L " + "ON source.id1 = L.id1 and source.id2 = L.id2"; - Table result = tEnv.sqlQuery(sqlQuery); - DataStream resultSet = tEnv.toAppendStream(result, Row.class); - resultSet.addSink(new StreamITCase.StringSink<>()); - env.execute(); + return tEnv.executeSql(sqlQuery).collect(); } } diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.java new file mode 100644 index 0000000000000..4efcb47b9808f --- /dev/null +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.jdbc.table; + +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.planner.utils.StreamTableTestUtil; +import org.apache.flink.table.planner.utils.TableTestBase; + +import org.junit.Test; + +/** + * Plan tests for JDBC connector, for example, testing projection push down. + */ +public class JdbcTablePlanTest extends TableTestBase { + + private final StreamTableTestUtil util = streamTestUtil(new TableConfig()); + + @Test + public void testProjectionPushDown() { + util.tableEnv().executeSql( + "CREATE TABLE jdbc (" + + "id BIGINT," + + "timestamp6_col TIMESTAMP(6)," + + "timestamp9_col TIMESTAMP(9)," + + "time_col TIME," + + "real_col FLOAT," + + "double_col DOUBLE," + + "decimal_col DECIMAL(10, 4)" + + ") WITH (" + + " 'connector'='jdbc'," + + " 'url'='jdbc:derby:memory:test'," + + " 'table-name'='test_table'" + + ")" + ); + util.verifyPlan("SELECT decimal_col, timestamp9_col, id FROM jdbc"); + } + +} diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java index 277191c452ae8..81156961a6e9e 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java @@ -44,7 +44,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; - /** * ITCase for {@link JdbcTableSource}. */ diff --git a/flink-connectors/flink-connector-jdbc/src/test/resources/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.xml b/flink-connectors/flink-connector-jdbc/src/test/resources/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.xml new file mode 100644 index 0000000000000..9219fc835add5 --- /dev/null +++ b/flink-connectors/flink-connector-jdbc/src/test/resources/org/apache/flink/connector/jdbc/table/JdbcTablePlanTest.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + From 0f072234d5cd30879b4e4845e69bee1a03cf1817 Mon Sep 17 00:00:00 2001 From: "Jiangjie (Becket) Qin" Date: Tue, 19 May 2020 23:44:17 +0800 Subject: [PATCH 065/773] [FLINK-12030][connector/kafka][tests] Check the topic existence after topic creation using KafkaConsumer. Signed-off-by: Jiangjie (Becket) Qin --- .../kafka/KafkaTestEnvironmentImpl.java | 43 ++++++++----------- .../kafka/KafkaTestEnvironmentImpl.java | 43 ++++++++----------- 2 files changed, 38 insertions(+), 48 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java index 322c3aa284a61..9ae751bb57a04 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java @@ -36,6 +36,7 @@ import org.apache.commons.collections.list.UnmodifiableList; import org.apache.commons.io.FileUtils; import org.apache.curator.test.TestingServer; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; @@ -43,6 +44,7 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.SecurityProtocol; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -322,32 +324,25 @@ public void createTestTopic(String topic, int numberOfPartitions, int replicatio // validate that the topic has been created final long deadline = System.nanoTime() + 30_000_000_000L; - do { - try { - if (config.isSecureMode()) { - //increase wait time since in Travis ZK timeout occurs frequently - int wait = zkTimeout / 100; - LOG.info("waiting for {} msecs before the topic {} can be checked", wait, topic); - Thread.sleep(wait); - } else { - Thread.sleep(100); + boolean topicCreated = false; + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, getBrokerConnectionString()); + props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + props.putAll(getSecureProperties()); + try (KafkaConsumer consumer = new KafkaConsumer<>(props)) { + do { + topicCreated = !consumer.partitionsFor(topic).isEmpty(); + if (!topicCreated) { + Thread.sleep(10); } - } catch (InterruptedException e) { - // restore interrupted state - } - // we could use AdminUtils.topicExists(zkUtils, topic) here, but it's results are - // not always correct. - - // create a new ZK utils connection - ZkUtils checkZKConn = getZkUtils(); - if (AdminUtils.topicExists(checkZKConn, topic)) { - checkZKConn.close(); - return; - } - checkZKConn.close(); + } while (!topicCreated && System.nanoTime() < deadline); + } catch (InterruptedException e) { + // do nothing. + } + if (!topicCreated) { + fail("Test topic could not be created"); } - while (System.nanoTime() < deadline); - fail("Test topic could not be created"); } @Override diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java index 478ce38886736..c846f164173a1 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestEnvironmentImpl.java @@ -36,6 +36,7 @@ import org.apache.commons.collections.list.UnmodifiableList; import org.apache.commons.io.FileUtils; import org.apache.curator.test.TestingServer; +import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; @@ -43,6 +44,7 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.SecurityProtocol; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,32 +159,25 @@ public void createTestTopic(String topic, int numberOfPartitions, int replicatio // validate that the topic has been created final long deadline = System.nanoTime() + 30_000_000_000L; - do { - try { - if (config.isSecureMode()) { - //increase wait time since in Travis ZK timeout occurs frequently - int wait = zkTimeout / 100; - LOG.info("waiting for {} msecs before the topic {} can be checked", wait, topic); - Thread.sleep(wait); - } else { - Thread.sleep(100); + boolean topicCreated = false; + Properties props = new Properties(); + props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, getBrokerConnectionString()); + props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + props.putAll(getSecureProperties()); + try (KafkaConsumer consumer = new KafkaConsumer<>(props)) { + do { + topicCreated = !consumer.partitionsFor(topic).isEmpty(); + if (!topicCreated) { + Thread.sleep(10); } - } catch (InterruptedException e) { - // restore interrupted state - } - // we could use AdminUtils.topicExists(zkUtils, topic) here, but it's results are - // not always correct. - - // create a new ZK utils connection - ZkUtils checkZKConn = getZkUtils(); - if (AdminUtils.topicExists(checkZKConn, topic)) { - checkZKConn.close(); - return; - } - checkZKConn.close(); + } while (!topicCreated && System.nanoTime() < deadline); + } catch (InterruptedException e) { + // do nothing. + } + if (!topicCreated) { + fail("Test topic could not be created"); } - while (System.nanoTime() < deadline); - fail("Test topic could not be created"); } @Override From fa731288518c8ebf66f40b4e0e9b1929546b6257 Mon Sep 17 00:00:00 2001 From: Yun Tang Date: Mon, 11 May 2020 13:45:45 +0800 Subject: [PATCH 066/773] [FLINK-8871][checkpoint] Support to cancel checkpoing via notification on task side --- .../fs/bucketing/BucketingSink.java | 4 + .../connectors/gcp/pubsub/PubSubSource.java | 4 + .../common/AcknowledgeOnCheckpoint.java | 4 + .../kafka/FlinkKafkaConsumerBase.java | 4 + .../kafka/KafkaConsumerTestBase.java | 4 + .../kafka/KafkaProducerTestBase.java | 4 + .../testutils/FailingIdentityMapper.java | 4 + .../kafka/testutils/IntegerSource.java | 4 + .../flink/streaming/tests/FailureMapper.java | 4 + .../HeavyDeploymentStressTestProgram.java | 4 + ...ickyAllocationAndLocalRecoveryTestJob.java | 4 + .../runtime/SavepointTaskStateManager.java | 5 + .../state/api/output/SnapshotUtilsTest.java | 4 + .../AbstractQueryableStateTestBase.java | 4 + .../jobgraph/tasks/AbstractInvokable.java | 12 + .../runtime/state/CheckpointListener.java | 8 + .../state/NoOpTaskLocalStateStoreImpl.java | 4 + .../runtime/state/TaskLocalStateStore.java | 6 + .../state/TaskLocalStateStoreImpl.java | 9 + .../runtime/state/TaskStateManagerImpl.java | 10 +- .../state/heap/HeapKeyedStateBackend.java | 5 + .../flink/runtime/taskmanager/Task.java | 28 ++ .../state/TaskLocalStateStoreImplTest.java | 23 +- .../state/TestTaskLocalStateStore.java | 19 ++ .../runtime/state/TestTaskStateManager.java | 11 + .../state/ttl/mock/MockKeyedStateBackend.java | 5 + .../state/RocksDBKeyedStateBackend.java | 7 + .../snapshot/RocksFullSnapshotStrategy.java | 5 + .../RocksIncrementalSnapshotStrategy.java | 7 + .../sink/TwoPhaseCommitSinkFunction.java | 4 + .../sink/filesystem/StreamingFileSink.java | 4 + .../MessageAcknowledgingSourceBase.java | 4 + .../api/operators/AbstractStreamOperator.java | 5 + .../operators/AbstractStreamOperatorV2.java | 5 + .../operators/StreamOperatorStateHandler.java | 6 + .../collect/CollectSinkFunction.java | 4 + .../tasks/AsyncCheckpointRunnable.java | 22 +- .../streaming/runtime/tasks/StreamTask.java | 7 + .../tasks/SubtaskCheckpointCoordinator.java | 12 + .../SubtaskCheckpointCoordinatorImpl.java | 198 +++++++++++- ...bstractUdfStreamOperatorLifecycleTest.java | 1 + .../tasks/LocalStateForwardingTest.java | 3 +- ...ckSubtaskCheckpointCoordinatorBuilder.java | 14 +- .../SubtaskCheckpointCoordinatorTest.java | 297 +++++++++++++++++- .../tasks/SynchronousCheckpointITCase.java | 6 + .../utils/FailingCollectionSource.java | 4 + .../stream/FsStreamingSinkITCaseBase.scala | 3 + .../streaming/util/FiniteTestSource.java | 4 + .../JobMasterStopWithSavepointIT.java | 5 + .../JobMasterTriggerSavepointITCase.java | 5 + .../CoStreamCheckpointingITCase.java | 4 + ...tinuousFileProcessingCheckpointITCase.java | 4 + .../KeyedStateCheckpointingITCase.java | 4 + .../StateCheckpointedITCase.java | 4 + .../StreamCheckpointNotifierITCase.java | 20 ++ .../UnalignedCheckpointITCase.java | 8 + .../ZooKeeperHighAvailabilityITCase.java | 4 + .../utils/AccumulatingIntegerSink.java | 4 + .../utils/CancellingIntegerSource.java | 4 + .../checkpointing/utils/FailingSource.java | 4 + .../jar/CheckpointedStreamingProgram.java | 4 + .../CheckpointingCustomKvStateProgram.java | 4 + ...nterpretDataStreamAsKeyedStreamITCase.java | 4 + 63 files changed, 877 insertions(+), 24 deletions(-) diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java index ec14ccefe47ae..78cefafd3ce7b 100644 --- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java +++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java @@ -732,6 +732,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkNotNull(restoredBucketStates, "The operator has not been properly initialized."); diff --git a/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/PubSubSource.java b/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/PubSubSource.java index 1472bb2f60e47..8007fc12f75af 100644 --- a/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/PubSubSource.java +++ b/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/PubSubSource.java @@ -193,6 +193,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { acknowledgeOnCheckpoint.notifyCheckpointComplete(checkpointId); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public List> snapshotState(long checkpointId, long timestamp) throws Exception { return acknowledgeOnCheckpoint.snapshotState(checkpointId, timestamp); diff --git a/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/common/AcknowledgeOnCheckpoint.java b/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/common/AcknowledgeOnCheckpoint.java index f538b697accb2..6b194b6862128 100644 --- a/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/common/AcknowledgeOnCheckpoint.java +++ b/flink-connectors/flink-connector-gcp-pubsub/src/main/java/org/apache/flink/streaming/connectors/gcp/pubsub/common/AcknowledgeOnCheckpoint.java @@ -82,6 +82,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { outstandingAcknowledgements = new AtomicInteger(numberOfAcknowledgementIds(acknowledgeIdsPerCheckpoint)); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public List> snapshotState(long checkpointId, long timestamp) throws Exception { acknowledgeIdsPerCheckpoint.add(new AcknowledgeIdsForCheckpoint<>(checkpointId, acknowledgeIdsForPendingCheckpoint)); diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index 84057b0847d79..733011f9b34fc 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -1035,6 +1035,10 @@ public final void notifyCheckpointComplete(long checkpointId) throws Exception { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + // ------------------------------------------------------------------------ // Kafka Consumer specific methods // ------------------------------------------------------------------------ diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java index 95688b21d3db3..d99f8d9a0516a 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java @@ -2249,6 +2249,10 @@ public void notifyCheckpointComplete(long checkpointId) { hasBeenCheckpointed = true; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public List snapshotState(long checkpointId, long timestamp) throws Exception { return Collections.singletonList(this.numElementsTotal); diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java index c0c98446b47c5..2b586fe746c3f 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java @@ -530,6 +530,10 @@ public T map(T value) throws Exception { public void notifyCheckpointComplete(long checkpointId) { } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { if (!triggeredShutdown) { diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java index bd412c911c9a5..9919f1ec866ab 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/FailingIdentityMapper.java @@ -94,6 +94,10 @@ public void notifyCheckpointComplete(long checkpointId) { this.hasBeenCheckpointed = true; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public List snapshotState(long checkpointId, long timestamp) throws Exception { return Collections.singletonList(numElementsTotal); diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/IntegerSource.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/IntegerSource.java index 25a3cead4e7b3..f471df495fd8d 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/IntegerSource.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/testutils/IntegerSource.java @@ -127,4 +127,8 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { blocker.notifyAll(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } diff --git a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java index a3a1c253fc070..458f9b2a6b659 100644 --- a/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java +++ b/flink-end-to-end-tests/flink-datastream-allround-test/src/main/java/org/apache/flink/streaming/tests/FailureMapper.java @@ -71,6 +71,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + private boolean isReachedFailureThreshold() { return numProcessedRecords >= numProcessedRecordsFailureThreshold && numCompleteCheckpoints >= numCompleteCheckpointsFailureThreshold diff --git a/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java b/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java index d65583f98afce..68b6d245c6a1f 100644 --- a/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java +++ b/flink-end-to-end-tests/flink-heavy-deployment-stress-test/src/main/java/org/apache/flink/deployment/HeavyDeploymentStressTestProgram.java @@ -146,5 +146,9 @@ public void cancel() { public void notifyCheckpointComplete(long checkpointId) { readyToFail = true; } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } } diff --git a/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java b/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java index b03791e26eb70..990baa9168b62 100644 --- a/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java +++ b/flink-end-to-end-tests/flink-local-recovery-and-allocation-test/src/main/java/org/apache/flink/streaming/tests/StickyAllocationAndLocalRecoveryTestJob.java @@ -378,6 +378,10 @@ public void notifyCheckpointComplete(long checkpointId) { failTask = currentSchedulingAndFailureInfo.failingTask; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + private boolean shouldTaskFailForThisAttempt() { RuntimeContext runtimeContext = getRuntimeContext(); int numSubtasks = runtimeContext.getNumberOfParallelSubtasks(); diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointTaskStateManager.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointTaskStateManager.java index 9563e40faa535..4fb15a2c9590e 100644 --- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointTaskStateManager.java +++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointTaskStateManager.java @@ -77,6 +77,11 @@ public void notifyCheckpointComplete(long checkpointId) { throw new UnsupportedOperationException(MSG); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + throw new UnsupportedOperationException(MSG); + } + @Override public void close() { } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java index 566828432a593..36b4d634dc512 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java @@ -130,6 +130,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { ACTUAL_ORDER_TRACKING.add("notifyCheckpointComplete"); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void setCurrentKey(Object key) { ACTUAL_ORDER_TRACKING.add("setCurrentKey"); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java index 83444e8a645f9..d9e3c91bfe80c 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java @@ -1124,6 +1124,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { LATEST_CHECKPOINT_ID.set(checkpointId); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java index 72586abd58555..af57f971a292a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java @@ -278,6 +278,18 @@ public Future notifyCheckpointCompleteAsync(long checkpointId) { throw new UnsupportedOperationException(String.format("notifyCheckpointCompleteAsync not supported by %s", this.getClass().getName())); } + /** + * Invoked when a checkpoint has been aborted, i.e., when the checkpoint coordinator has received a decline message + * from one task and try to abort the targeted checkpoint by notification. + * + * @param checkpointId The ID of the checkpoint that is aborted. + * + * @return future that completes when the notification has been processed by the task. + */ + public Future notifyCheckpointAbortAsync(long checkpointId) { + throw new UnsupportedOperationException(String.format("notifyCheckpointAbortAsync not supported by %s", this.getClass().getName())); + } + public void dispatchOperatorEvent(OperatorID operator, SerializedValue event) throws FlinkException { throw new UnsupportedOperationException("dispatchOperatorEvent not supported by " + getClass().getName()); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java index 0c99316d932c9..13c8e397e91cc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/CheckpointListener.java @@ -38,4 +38,12 @@ public interface CheckpointListener { * @throws Exception */ void notifyCheckpointComplete(long checkpointId) throws Exception; + + /** + * This method is called as a notification once a distributed checkpoint has been aborted. + * + * @param checkpointId The ID of the checkpoint that has been aborted. + * @throws Exception + */ + void notifyCheckpointAborted(long checkpointId) throws Exception; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/NoOpTaskLocalStateStoreImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/NoOpTaskLocalStateStoreImpl.java index 11841a1407e55..aece4aa3f406b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/NoOpTaskLocalStateStoreImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/NoOpTaskLocalStateStoreImpl.java @@ -65,6 +65,10 @@ public TaskStateSnapshot retrieveLocalState(long checkpointID) { public void confirmCheckpoint(long confirmedCheckpointId) { } + @Override + public void abortCheckpoint(long abortedCheckpointId) { + } + @Override public void pruneMatchingCheckpoints(LongPredicate matcher) { } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java index b0d8a824c5372..78f5068eaecf7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java @@ -67,6 +67,12 @@ void storeLocalState( */ void confirmCheckpoint(long confirmedCheckpointId); + /** + * Notifies that the checkpoint with the given id was confirmed as aborted. This prunes the checkpoint history + * and removes states with a checkpoint id that is equal to the newly aborted checkpoint id. + */ + void abortCheckpoint(long abortedCheckpointId); + /** * Remove all checkpoints from the store that match the given predicate. * @param matcher the predicate that selects the checkpoints for pruning. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java index a57a7efd08a6c..52d7811f78276 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java @@ -224,6 +224,15 @@ public void confirmCheckpoint(long confirmedCheckpointId) { } + @Override + public void abortCheckpoint(long abortedCheckpointId) { + + LOG.debug("Received abort information for checkpoint {} in subtask ({} - {} - {}). Starting to prune history.", + abortedCheckpointId, jobID, jobVertexID, subtaskIndex); + + pruneCheckpoints(snapshotCheckpointId -> snapshotCheckpointId == abortedCheckpointId, false); + } + @Override public void pruneMatchingCheckpoints(@Nonnull LongPredicate matcher) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java index a42e8085b8d50..f2ecde0e1e36c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java @@ -180,13 +180,21 @@ public ChannelStateReader getChannelStateReader() { } /** - * Tracking when local state can be disposed. + * Tracking when local state can be confirmed and disposed. */ @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { localStateStore.confirmCheckpoint(checkpointId); } + /** + * Tracking when some local state can be disposed. + */ + @Override + public void notifyCheckpointAborted(long checkpointId) { + localStateStore.abortCheckpoint(checkpointId); + } + @Override public void close() throws Exception { channelStateReader.close(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index 7e184ab958814..68828bab00ddc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -311,6 +311,11 @@ public void notifyCheckpointComplete(long checkpointId) { //Nothing to do } + @Override + public void notifyCheckpointAborted(long checkpointId) { + // nothing to do + } + @Override public void applyToAllKeys( final N namespace, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index 2e3c9b6acf0aa..bd4671f5bb31b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -1218,6 +1218,34 @@ public void notifyCheckpointComplete(final long checkpointID) { } } + @Override + public void notifyCheckpointAborted(final long checkpointID) { + final AbstractInvokable invokable = this.invokable; + + if (executionState == ExecutionState.RUNNING && invokable != null) { + try { + invokable.notifyCheckpointAbortAsync(checkpointID); + } + catch (RejectedExecutionException ex) { + // This may happen if the mailbox is closed. It means that the task is shutting down, so we just ignore it. + LOG.debug( + "Notify checkpoint abort {} for {} ({}) was rejected by the mailbox", + checkpointID, taskNameWithSubtask, executionId); + } + catch (Throwable t) { + if (getExecutionState() == ExecutionState.RUNNING) { + // fail task if checkpoint aborted notification failed. + failExternally(new RuntimeException( + "Error while aborting checkpoint", + t)); + } + } + } + else { + LOG.info("Ignoring checkpoint aborted notification for non-running task {}.", taskNameWithSubtask); + } + } + /** * Dispatches an operator event to the invokable task. * diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java index 75317834b183a..784015be84600 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java @@ -153,6 +153,21 @@ public void confirmCheckpoint() throws Exception { checkStoredAsExpected(taskStateSnapshots, confirmed, chkCount); } + /** + * Tests pruning of target previous checkpoints if that checkpoint is aborted. + */ + @Test + public void abortCheckpoint() throws Exception { + + final int chkCount = 4; + final int aborted = chkCount - 2; + List taskStateSnapshots = storeStates(chkCount); + taskLocalStateStore.abortCheckpoint(aborted); + checkPrunedAndDiscarded(taskStateSnapshots, aborted, aborted + 1); + checkStoredAsExpected(taskStateSnapshots, 0, aborted); + checkStoredAsExpected(taskStateSnapshots, aborted + 1, chkCount); + } + /** * Tests that disposal of a {@link TaskLocalStateStoreImpl} works and discards all local states. */ @@ -167,16 +182,16 @@ public void dispose() throws Exception { checkPrunedAndDiscarded(taskStateSnapshots, 0, chkCount); } - private void checkStoredAsExpected(List history, int off, int len) throws Exception { - for (int i = off; i < len; ++i) { + private void checkStoredAsExpected(List history, int start, int end) throws Exception { + for (int i = start; i < end; ++i) { TaskStateSnapshot expected = history.get(i); Assert.assertTrue(expected == taskLocalStateStore.retrieveLocalState(i)); Mockito.verify(expected, Mockito.never()).discardState(); } } - private void checkPrunedAndDiscarded(List history, int off, int len) throws Exception { - for (int i = off; i < len; ++i) { + private void checkPrunedAndDiscarded(List history, int start, int end) throws Exception { + for (int i = start; i < end; ++i) { Assert.assertNull(taskLocalStateStore.retrieveLocalState(i)); Mockito.verify(history.get(i)).discardState(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java index 2ade3e6d14262..e92a34affd8f5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java @@ -104,6 +104,25 @@ public void confirmCheckpoint(long confirmedCheckpointId) { } } + @Override + public void abortCheckpoint(long abortedCheckpointId) { + Preconditions.checkState(!disposed); + Iterator> iterator = taskStateSnapshotsByCheckpointID.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getKey() == abortedCheckpointId) { + iterator.remove(); + try { + entry.getValue().discardState(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else if (entry.getKey() > abortedCheckpointId){ + break; + } + } + } + @Override public void pruneMatchingCheckpoints(LongPredicate matcher) { taskStateSnapshotsByCheckpointID.keySet().removeIf(matcher::test); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskStateManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskStateManager.java index 1ce41b396ca9c..ae6c022b8350e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskStateManager.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskStateManager.java @@ -47,6 +47,7 @@ public class TestTaskStateManager implements TaskStateManager { private long reportedCheckpointId; private long notifiedCompletedCheckpointId; + private long notifiedAbortedCheckpointId; private JobID jobId; private ExecutionAttemptID executionAttemptID; @@ -88,6 +89,7 @@ public TestTaskStateManager( this.taskManagerTaskStateSnapshotsByCheckpointId = new HashMap<>(); this.reportedCheckpointId = -1L; this.notifiedCompletedCheckpointId = -1L; + this.notifiedAbortedCheckpointId = -1L; } @Override @@ -175,6 +177,11 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { this.notifiedCompletedCheckpointId = checkpointId; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + this.notifiedAbortedCheckpointId = checkpointId; + } + public JobID getJobId() { return jobId; } @@ -227,6 +234,10 @@ public long getNotifiedCompletedCheckpointId() { return notifiedCompletedCheckpointId; } + public long getNotifiedAbortedCheckpointId() { + return notifiedAbortedCheckpointId; + } + public void setReportedCheckpointId(long reportedCheckpointId) { this.reportedCheckpointId = reportedCheckpointId; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java index 5c82b80e352d6..5cb6866e813de 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/ttl/mock/MockKeyedStateBackend.java @@ -166,6 +166,11 @@ public void notifyCheckpointComplete(long checkpointId) { // noop } + @Override + public void notifyCheckpointAborted(long checkpointId) { + // noop + } + @Override public Stream getKeys(String state, N namespace) { return stateValues.get(state).entrySet().stream() diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 2ddb79b207f5d..61d8688dfb566 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -464,6 +464,13 @@ public void notifyCheckpointComplete(long completedCheckpointId) throws Exceptio } } + @Override + public void notifyCheckpointAborted(long checkpointId) throws Exception { + checkpointSnapshotStrategy.notifyCheckpointAborted(checkpointId); + + savepointSnapshotStrategy.notifyCheckpointAborted(checkpointId); + } + /** * Registers a k/v state information, which includes its state id, type, RocksDB column family handle, and serializers. * diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksFullSnapshotStrategy.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksFullSnapshotStrategy.java index 10852837ffc0d..751eed815781e 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksFullSnapshotStrategy.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksFullSnapshotStrategy.java @@ -150,6 +150,11 @@ public void notifyCheckpointComplete(long checkpointId) { // nothing to do. } + @Override + public void notifyCheckpointAborted(long checkpointId) { + // nothing to do. + } + private SupplierWithException createCheckpointStreamSupplier( long checkpointId, CheckpointStreamFactory primaryStreamFactory, diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java index 23f4574438593..77b13428bff72 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java @@ -179,6 +179,13 @@ public void notifyCheckpointComplete(long completedCheckpointId) { } } + @Override + public void notifyCheckpointAborted(long abortedCheckpointId) { + synchronized (materializedSstFiles) { + materializedSstFiles.keySet().remove(abortedCheckpointId); + } + } + @Nonnull private SnapshotDirectory prepareLocalSnapshotDirectory(long checkpointId) throws IOException { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java index 6a42fb964da00..361bf3f746bfe 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java @@ -304,6 +304,10 @@ public final void notifyCheckpointComplete(long checkpointId) throws Exception { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { // this is like the pre-commit of a 2-phase-commit transaction diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java index 54abac4d73a84..0962799b20802 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java @@ -423,6 +423,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { this.helper.commitUpToCheckpoint(checkpointId); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(helper != null, "sink has not been initialized"); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java index ffb20154d1cff..3a2a5cad982f0 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/MessageAcknowledgingSourceBase.java @@ -241,4 +241,8 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java index a6982548c983c..a249d5e78e0b4 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java @@ -344,6 +344,11 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { stateHandler.notifyCheckpointComplete(checkpointId); } + @Override + public void notifyCheckpointAborted(long checkpointId) throws Exception { + stateHandler.notifyCheckpointAborted(checkpointId); + } + // ------------------------------------------------------------------------ // Properties and Services // ------------------------------------------------------------------------ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java index 56533db70527e..819b254369502 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java @@ -290,6 +290,11 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { stateHandler.notifyCheckpointComplete(checkpointId); } + @Override + public void notifyCheckpointAborted(long checkpointId) throws Exception { + stateHandler.notifyCheckpointAborted(checkpointId); + } + // ------------------------------------------------------------------------ // Properties and Services // ------------------------------------------------------------------------ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperatorStateHandler.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperatorStateHandler.java index 99123b1038cd4..ed03907042f34 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperatorStateHandler.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperatorStateHandler.java @@ -222,6 +222,12 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } } + public void notifyCheckpointAborted(long checkpointId) throws Exception { + if (keyedStateBackend != null) { + keyedStateBackend.notifyCheckpointAborted(checkpointId); + } + } + @SuppressWarnings("unchecked") public KeyedStateBackend getKeyedStateBackend() { return (KeyedStateBackend) keyedStateBackend; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkFunction.java index 64a3ffed1e263..a687283acebe5 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkFunction.java @@ -287,6 +287,10 @@ public void notifyCheckpointComplete(long checkpointId) { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + public void setOperatorEventGateway(OperatorEventGateway eventGateway) { this.eventGateway = eventGateway; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsyncCheckpointRunnable.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsyncCheckpointRunnable.java index b09c5a936d13a..e89e9624ad50e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsyncCheckpointRunnable.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/AsyncCheckpointRunnable.java @@ -17,7 +17,6 @@ package org.apache.flink.streaming.runtime.tasks; -import org.apache.flink.core.fs.CloseableRegistry; import org.apache.flink.core.fs.FileSystemSafetyNet; import org.apache.flink.runtime.checkpoint.CheckpointMetaData; import org.apache.flink.runtime.checkpoint.CheckpointMetrics; @@ -36,6 +35,7 @@ import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -46,10 +46,11 @@ final class AsyncCheckpointRunnable implements Runnable, Closeable { public static final Logger LOG = LoggerFactory.getLogger(AsyncCheckpointRunnable.class); private final String taskName; - private final CloseableRegistry closeableRegistry; + private final Consumer registerConsumer; + private final Consumer unregisterConsumer; private final Environment taskEnvironment; - private enum AsyncCheckpointState { + enum AsyncCheckpointState { RUNNING, DISCARDED, COMPLETED @@ -70,7 +71,8 @@ private enum AsyncCheckpointState { Future channelWrittenFuture, long asyncStartNanos, String taskName, - CloseableRegistry closeableRegistry, + Consumer register, + Consumer unregister, Environment taskEnvironment, AsyncExceptionHandler asyncExceptionHandler) { @@ -80,7 +82,8 @@ private enum AsyncCheckpointState { this.channelWrittenFuture = checkNotNull(channelWrittenFuture); this.asyncStartNanos = asyncStartNanos; this.taskName = checkNotNull(taskName); - this.closeableRegistry = checkNotNull(closeableRegistry); + this.registerConsumer = register; + this.unregisterConsumer = unregister; this.taskEnvironment = checkNotNull(taskEnvironment); this.asyncExceptionHandler = checkNotNull(asyncExceptionHandler); } @@ -89,7 +92,7 @@ private enum AsyncCheckpointState { public void run() { FileSystemSafetyNet.initializeSafetyNetForThread(); try { - closeableRegistry.registerCloseable(this); + registerConsumer.accept(this); TaskStateSnapshot jobManagerTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size()); TaskStateSnapshot localTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size()); @@ -140,7 +143,7 @@ public void run() { } handleExecutionException(e); } finally { - closeableRegistry.unregisterCloseable(this); + unregisterConsumer.accept(this); FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread(); } } @@ -229,6 +232,10 @@ public void close() { } } + long getCheckpointId() { + return checkpointMetaData.getCheckpointId(); + } + private void cleanup() throws Exception { LOG.debug( "Cleanup AsyncCheckpointRunnable for checkpoint {} of {}.", @@ -259,4 +266,5 @@ private void logFailedCleanupAttempt() { taskName, checkpointMetaData.getCheckpointId()); } + } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java index 0bcbc300b9a74..1c346b1a203dc 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java @@ -933,6 +933,13 @@ private void notifyCheckpointComplete(long checkpointId) throws Exception { } } + @Override + public Future notifyCheckpointAbortAsync(long checkpointId) { + return mailboxProcessor.getMailboxExecutor(TaskMailbox.MAX_PRIORITY).submit( + () -> subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, this::isRunning), + "checkpoint %d aborted", checkpointId); + } + private void tryShutdownTimerService() { if (!timerService.isTerminated()) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinator.java index d7352c67b89d9..29227352c8efa 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinator.java @@ -65,4 +65,16 @@ void notifyCheckpointComplete( long checkpointId, OperatorChain operatorChain, Supplier isRunning) throws Exception; + + /** + * Notified on the task side once a distributed checkpoint has been aborted. + * + * @param checkpointId The checkpoint id to notify as been completed. + * @param operatorChain The chain of operators executed by the task. + * @param isRunning Whether the task is running. + */ + void notifyCheckpointAborted( + long checkpointId, + OperatorChain operatorChain, + Supplier isRunning) throws Exception; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java index 8a55aa658f325..7508a16805726 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java @@ -17,6 +17,7 @@ package org.apache.flink.streaming.runtime.tasks; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.fs.CloseableRegistry; import org.apache.flink.runtime.checkpoint.CheckpointException; import org.apache.flink.runtime.checkpoint.CheckpointFailureReason; @@ -41,19 +42,31 @@ import org.apache.flink.runtime.state.SnapshotResult; import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.IOUtils; +import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.BiFunctionWithException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.concurrent.GuardedBy; + import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.apache.flink.runtime.checkpoint.CheckpointType.CHECKPOINT; @@ -62,10 +75,10 @@ class SubtaskCheckpointCoordinatorImpl implements SubtaskCheckpointCoordinator { private static final Logger LOG = LoggerFactory.getLogger(SubtaskCheckpointCoordinatorImpl.class); + private static final int DEFAULT_MAX_RECORD_ABORTED_CHECKPOINTS = 128; private final CachingCheckpointStorageWorkerView checkpointStorage; private final String taskName; - private final CloseableRegistry closeableRegistry; private final ExecutorService executorService; private final Environment env; private final AsyncExceptionHandler asyncExceptionHandler; @@ -73,6 +86,19 @@ class SubtaskCheckpointCoordinatorImpl implements SubtaskCheckpointCoordinator { private final StreamTaskActionExecutor actionExecutor; private final boolean unalignedCheckpointEnabled; private final BiFunctionWithException, IOException> prepareInputSnapshot; + /** The IDs of the checkpoint for which we are notified aborted. */ + private final Set abortedCheckpointIds; + private long lastCheckpointId; + + /** Lock that guards state of AsyncCheckpointRunnable registry. **/ + private final Object lock; + + @GuardedBy("lock") + private final Map checkpoints; + + /** Indicates if this registry is closed. */ + @GuardedBy("lock") + private boolean closed; SubtaskCheckpointCoordinatorImpl( CheckpointStorageWorkerView checkpointStorage, @@ -84,9 +110,34 @@ class SubtaskCheckpointCoordinatorImpl implements SubtaskCheckpointCoordinator { AsyncExceptionHandler asyncExceptionHandler, boolean unalignedCheckpointEnabled, BiFunctionWithException, IOException> prepareInputSnapshot) throws IOException { + this(checkpointStorage, + taskName, + actionExecutor, + closeableRegistry, + executorService, + env, + asyncExceptionHandler, + unalignedCheckpointEnabled, + prepareInputSnapshot, + DEFAULT_MAX_RECORD_ABORTED_CHECKPOINTS); + } + + @VisibleForTesting + SubtaskCheckpointCoordinatorImpl( + CheckpointStorageWorkerView checkpointStorage, + String taskName, + StreamTaskActionExecutor actionExecutor, + CloseableRegistry closeableRegistry, + ExecutorService executorService, + Environment env, + AsyncExceptionHandler asyncExceptionHandler, + boolean unalignedCheckpointEnabled, + BiFunctionWithException, IOException> prepareInputSnapshot, + int maxRecordAbortedCheckpoints) throws IOException { this.checkpointStorage = new CachingCheckpointStorageWorkerView(checkNotNull(checkpointStorage)); this.taskName = checkNotNull(taskName); - this.closeableRegistry = checkNotNull(closeableRegistry); + this.checkpoints = new HashMap<>(); + this.lock = new Object(); this.executorService = checkNotNull(executorService); this.env = checkNotNull(env); this.asyncExceptionHandler = checkNotNull(asyncExceptionHandler); @@ -94,7 +145,10 @@ class SubtaskCheckpointCoordinatorImpl implements SubtaskCheckpointCoordinator { this.channelStateWriter = unalignedCheckpointEnabled ? openChannelStateWriter() : ChannelStateWriter.NO_OP; this.unalignedCheckpointEnabled = unalignedCheckpointEnabled; this.prepareInputSnapshot = prepareInputSnapshot; - this.closeableRegistry.registerCloseable(this); + this.abortedCheckpointIds = createAbortedCheckpointSetWithLimitSize(maxRecordAbortedCheckpoints); + this.lastCheckpointId = -1L; + closeableRegistry.registerCloseable(this); + this.closed = false; } private ChannelStateWriter openChannelStateWriter() { @@ -144,6 +198,15 @@ public void checkpointState( // We generally try to emit the checkpoint barrier as soon as possible to not affect downstream // checkpoint alignments + // Step (0): Record the last triggered checkpointId. + Preconditions.checkArgument(lastCheckpointId < metadata.getCheckpointId(), String.format( + "Unexpected current checkpoint-id: %s vs last checkpoint-id: %s", metadata.getCheckpointId(), lastCheckpointId)); + lastCheckpointId = metadata.getCheckpointId(); + if (checkAndClearAbortedStatus(metadata.getCheckpointId())) { + LOG.info("Checkpoint {} has been notified as aborted, would not trigger any checkpoint.", metadata.getCheckpointId()); + return; + } + // Step (1): Prepare the checkpoint, allow operators to do some pre-barrier work. // The pre-barrier work should be nothing or minimal in the common case. operatorChain.prepareSnapshotPreBarrier(metadata.getCheckpointId()); @@ -188,6 +251,106 @@ public void notifyCheckpointComplete(long checkpointId, OperatorChain oper env.getTaskStateManager().notifyCheckpointComplete(checkpointId); } + @Override + public void notifyCheckpointAborted(long checkpointId, OperatorChain operatorChain, Supplier isRunning) throws Exception { + + Exception previousException = null; + if (isRunning.get()) { + LOG.debug("Notification of aborted checkpoint for task {}", taskName); + + boolean canceled = cancelAsyncCheckpointRunnable(checkpointId); + + if (!canceled) { + if (checkpointId > lastCheckpointId) { + // only record checkpoints that have not triggered on task side. + abortedCheckpointIds.add(checkpointId); + } + } + + for (StreamOperatorWrapper operatorWrapper : operatorChain.getAllOperators(true)) { + try { + operatorWrapper.getStreamOperator().notifyCheckpointAborted(checkpointId); + } catch (Exception e) { + previousException = e; + } + } + + } else { + LOG.debug("Ignoring notification of aborted checkpoint for not-running task {}", taskName); + } + + env.getTaskStateManager().notifyCheckpointAborted(checkpointId); + ExceptionUtils.tryRethrowException(previousException); + } + + @Override + public void close() throws IOException { + List asyncCheckpointRunnables = null; + synchronized (lock) { + if (!closed) { + closed = true; + asyncCheckpointRunnables = new ArrayList<>(checkpoints.values()); + checkpoints.clear(); + } + } + IOUtils.closeAllQuietly(asyncCheckpointRunnables); + channelStateWriter.close(); + } + + @VisibleForTesting + int getAsyncCheckpointRunnableSize() { + synchronized (lock) { + return checkpoints.size(); + } + } + + @VisibleForTesting + int getAbortedCheckpointSize() { + return abortedCheckpointIds.size(); + } + + private boolean checkAndClearAbortedStatus(long checkpointId) { + return abortedCheckpointIds.remove(checkpointId); + } + + private void registerAsyncCheckpointRunnable(long checkpointId, AsyncCheckpointRunnable asyncCheckpointRunnable) throws IOException { + StringBuilder exceptionMessage = new StringBuilder("Cannot register Closeable, "); + synchronized (lock) { + if (!closed) { + if (!checkpoints.containsKey(checkpointId)) { + checkpoints.put(checkpointId, asyncCheckpointRunnable); + return; + } else { + exceptionMessage.append("async checkpoint ").append(checkpointId).append(" runnable has been register. "); + } + } else { + exceptionMessage.append("this subtaskCheckpointCoordinator is already closed. "); + } + } + + IOUtils.closeQuietly(asyncCheckpointRunnable); + throw new IOException(exceptionMessage.append("Closing argument.").toString()); + } + + private boolean unregisterAsyncCheckpointRunnable(long checkpointId) { + synchronized (lock) { + return checkpoints.remove(checkpointId) != null; + } + } + + /** + * Cancel the async checkpoint runnable with given checkpoint id. + * If given checkpoint id is not registered, return false, otherwise return true. + */ + private boolean cancelAsyncCheckpointRunnable(long checkpointId) { + AsyncCheckpointRunnable asyncCheckpointRunnable; + synchronized (lock) { + asyncCheckpointRunnable = checkpoints.remove(checkpointId); + } + IOUtils.closeQuietly(asyncCheckpointRunnable); + return asyncCheckpointRunnable != null; + } + private void cleanup( Map operatorSnapshotsInProgress, CheckpointMetaData metadata, @@ -251,11 +414,26 @@ private void finishAndReportAsync(Map snaps channelWrittenFuture, System.nanoTime(), taskName, - closeableRegistry, + registerConsumer(), + unregisterConsumer(), env, asyncExceptionHandler)); } + private Consumer registerConsumer() { + return asyncCheckpointRunnable -> { + try { + registerAsyncCheckpointRunnable(asyncCheckpointRunnable.getCheckpointId(), asyncCheckpointRunnable); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + private Consumer unregisterConsumer() { + return asyncCheckpointRunnable -> unregisterAsyncCheckpointRunnable(asyncCheckpointRunnable.getCheckpointId()); + } + private boolean takeSnapshotSync( Map operatorSnapshotsInProgress, CheckpointMetaData checkpointMetaData, @@ -342,9 +520,15 @@ private OperatorSnapshotFutures buildOperatorSnapshotFutures( return snapshotInProgress; } - @Override - public void close() throws IOException { - channelStateWriter.close(); + private Set createAbortedCheckpointSetWithLimitSize(int maxRecordAbortedCheckpoints) { + return Collections.newSetFromMap(new LinkedHashMap() { + private static final long serialVersionUID = 1L; + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > maxRecordAbortedCheckpoints; + } + }); } // Caches checkpoint output stream factories to prevent multiple output stream per checkpoint. diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperatorLifecycleTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperatorLifecycleTest.java index 5fb8830ef8a38..46dfbde99e492 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperatorLifecycleTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperatorLifecycleTest.java @@ -93,6 +93,7 @@ public class AbstractUdfStreamOperatorLifecycleTest { "getMetricGroup[], " + "getOperatorID[], " + "initializeState[interface org.apache.flink.streaming.api.operators.StreamTaskStateInitializer], " + + "notifyCheckpointAborted[long], " + "notifyCheckpointComplete[long], " + "open[], " + "prepareSnapshotPreBarrier[long], " + diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/LocalStateForwardingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/LocalStateForwardingTest.java index 8d4fae2aca817..3e0703fa47d40 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/LocalStateForwardingTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/LocalStateForwardingTest.java @@ -118,7 +118,8 @@ public void testReportingFromSnapshotToTaskStateManager() throws Exception { CompletableFuture.completedFuture(null), 0L, testStreamTask.getName(), - testStreamTask.getCancelables(), + asyncCheckpointRunnable -> {}, + asyncCheckpointRunnable -> {}, testStreamTask.getEnvironment(), testStreamTask); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MockSubtaskCheckpointCoordinatorBuilder.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MockSubtaskCheckpointCoordinatorBuilder.java index d3898db10e46b..e0db33f6de337 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MockSubtaskCheckpointCoordinatorBuilder.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MockSubtaskCheckpointCoordinatorBuilder.java @@ -47,6 +47,7 @@ public class MockSubtaskCheckpointCoordinatorBuilder { private ExecutorService executorService = Executors.newDirectExecutorService(); private BiFunctionWithException, IOException> prepareInputSnapshot = (channelStateWriter, aLong) -> FutureUtils.completedVoidFuture(); private boolean unalignedCheckpointEnabled; + private int maxRecordAbortedCheckpoints = 10; public MockSubtaskCheckpointCoordinatorBuilder setEnvironment(Environment environment) { this.environment = environment; @@ -58,6 +59,16 @@ public MockSubtaskCheckpointCoordinatorBuilder setPrepareInputSnapshot(BiFunctio return this; } + public MockSubtaskCheckpointCoordinatorBuilder setExecutor(ExecutorService executor) { + this.executorService = executor; + return this; + } + + public MockSubtaskCheckpointCoordinatorBuilder setMaxRecordAbortedCheckpoints(int maxRecordAbortedCheckpoints) { + this.maxRecordAbortedCheckpoints = maxRecordAbortedCheckpoints; + return this; + } + public MockSubtaskCheckpointCoordinatorBuilder setUnalignedCheckpointEnabled(boolean unalignedCheckpointEnabled) { this.unalignedCheckpointEnabled = unalignedCheckpointEnabled; return this; @@ -83,7 +94,8 @@ SubtaskCheckpointCoordinator build() throws IOException { environment, asyncExceptionHandler, unalignedCheckpointEnabled, - prepareInputSnapshot); + prepareInputSnapshot, + maxRecordAbortedCheckpoints); } private static class NonHandleAsyncException implements AsyncExceptionHandler { diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorTest.java index efe9185318b4f..94bb92f37ea1e 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorTest.java @@ -18,21 +18,45 @@ package org.apache.flink.streaming.runtime.tasks; +import org.apache.flink.core.testutils.OneShotLatch; +import org.apache.flink.metrics.MetricGroup; import org.apache.flink.runtime.checkpoint.CheckpointMetaData; import org.apache.flink.runtime.checkpoint.CheckpointMetrics; import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.io.network.api.writer.NonRecordWriter; +import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.operators.testutils.DummyEnvironment; import org.apache.flink.runtime.operators.testutils.MockEnvironment; import org.apache.flink.runtime.state.CheckpointStorageLocationReference; +import org.apache.flink.runtime.state.CheckpointStreamFactory; +import org.apache.flink.runtime.state.DoneFuture; +import org.apache.flink.runtime.state.KeyedStateHandle; +import org.apache.flink.runtime.state.SnapshotResult; import org.apache.flink.runtime.state.TestTaskStateManager; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; +import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTaskTest.NoOpStreamTask; import org.apache.flink.streaming.util.MockStreamTaskBuilder; +import org.apache.flink.util.ExceptionUtils; import org.junit.Test; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.TimeUnit; + import static org.apache.flink.runtime.checkpoint.CheckpointType.SAVEPOINT; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -48,9 +72,7 @@ public void testNotifyCheckpointComplete() throws Exception { .setEnvironment(mockEnvironment) .build(); - final OperatorChain operatorChain = new OperatorChain<>( - new MockStreamTaskBuilder(new DummyEnvironment()).build(), - new NonRecordWriter<>()); + final OperatorChain operatorChain = getOperatorChain(mockEnvironment); long checkpointId = 42L; { @@ -82,4 +104,273 @@ public void testSkipChannelStateForSavepoints() throws Exception { new OperatorChain<>(new NoOpStreamTask<>(new DummyEnvironment()), new NonRecordWriter<>()), () -> false); } + + @Test + public void testNotifyCheckpointAbortedManyTimes() throws Exception { + MockEnvironment mockEnvironment = MockEnvironment.builder().build(); + int maxRecordAbortedCheckpoints = 256; + SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder() + .setEnvironment(mockEnvironment) + .setMaxRecordAbortedCheckpoints(maxRecordAbortedCheckpoints) + .build(); + + final OperatorChain operatorChain = getOperatorChain(mockEnvironment); + + long notifyAbortedTimes = maxRecordAbortedCheckpoints + 42; + for (int i = 1; i < notifyAbortedTimes; i++) { + subtaskCheckpointCoordinator.notifyCheckpointAborted(i, operatorChain, () -> true); + assertEquals(Math.min(maxRecordAbortedCheckpoints, i), subtaskCheckpointCoordinator.getAbortedCheckpointSize()); + } + } + + @Test + public void testNotifyCheckpointAbortedBeforeAsyncPhase() throws Exception { + TestTaskStateManager stateManager = new TestTaskStateManager(); + MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build(); + SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder() + .setEnvironment(mockEnvironment) + .setUnalignedCheckpointEnabled(true) + .build(); + + CheckpointOperator checkpointOperator = new CheckpointOperator(new OperatorSnapshotFutures()); + + final OperatorChain> operatorChain = operatorChain(checkpointOperator); + + long checkpointId = 42L; + // notify checkpoint aborted before execution. + subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true); + assertEquals(1, subtaskCheckpointCoordinator.getAbortedCheckpointSize()); + + subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation()); + subtaskCheckpointCoordinator.checkpointState( + new CheckpointMetaData(checkpointId, System.currentTimeMillis()), + CheckpointOptions.forCheckpointWithDefaultLocation(), + new CheckpointMetrics(), + operatorChain, + () -> true); + assertFalse(checkpointOperator.isCheckpointed()); + assertEquals(-1, stateManager.getReportedCheckpointId()); + assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize()); + assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize()); + } + + @Test + public void testNotifyCheckpointAbortedDuringAsyncPhase() throws Exception { + MockEnvironment mockEnvironment = MockEnvironment.builder().build(); + SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder() + .setEnvironment(mockEnvironment) + .setExecutor(Executors.newSingleThreadExecutor()) + .setUnalignedCheckpointEnabled(true) + .build(); + + final BlockingRunnableFuture rawKeyedStateHandleFuture = new BlockingRunnableFuture(); + OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures( + DoneFuture.of(SnapshotResult.empty()), + rawKeyedStateHandleFuture, + DoneFuture.of(SnapshotResult.empty()), + DoneFuture.of(SnapshotResult.empty()), + DoneFuture.of(SnapshotResult.empty()), + DoneFuture.of(SnapshotResult.empty())); + + final OperatorChain> operatorChain = operatorChain(new CheckpointOperator(operatorSnapshotResult)); + + long checkpointId = 42L; + subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation()); + subtaskCheckpointCoordinator.checkpointState( + new CheckpointMetaData(checkpointId, System.currentTimeMillis()), + CheckpointOptions.forCheckpointWithDefaultLocation(), + new CheckpointMetrics(), + operatorChain, + () -> true); + rawKeyedStateHandleFuture.awaitRun(); + assertEquals(1, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize()); + assertFalse(rawKeyedStateHandleFuture.isCancelled()); + + subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true); + assertTrue(rawKeyedStateHandleFuture.isCancelled()); + assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize()); + } + + @Test + public void testNotifyCheckpointAbortedAfterAsyncPhase() throws Exception { + TestTaskStateManager stateManager = new TestTaskStateManager(); + MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build(); + SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder() + .setEnvironment(mockEnvironment) + .build(); + + final OperatorChain operatorChain = getOperatorChain(mockEnvironment); + + long checkpointId = 42L; + subtaskCheckpointCoordinator.checkpointState( + new CheckpointMetaData(checkpointId, System.currentTimeMillis()), + CheckpointOptions.forCheckpointWithDefaultLocation(), + new CheckpointMetrics(), + operatorChain, + () -> true); + subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true); + assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize()); + assertEquals(checkpointId, stateManager.getNotifiedAbortedCheckpointId()); + } + + private OperatorChain getOperatorChain(MockEnvironment mockEnvironment) throws Exception { + return new OperatorChain<>( + new MockStreamTaskBuilder(mockEnvironment).build(), + new NonRecordWriter<>()); + } + + private OperatorChain> operatorChain(OneInputStreamOperator... streamOperators) throws Exception { + return OperatorChainTest.setupOperatorChain(streamOperators); + } + + private static final class BlockingRunnableFuture implements RunnableFuture> { + + private final CompletableFuture> future = new CompletableFuture<>(); + + private final OneShotLatch signalRunLatch = new OneShotLatch(); + + private final CountDownLatch countDownLatch; + + private final SnapshotResult value; + + private BlockingRunnableFuture() { + // count down twice to wait for notify checkpoint aborted to cancel. + this.countDownLatch = new CountDownLatch(2); + this.value = SnapshotResult.empty(); + } + + @Override + public void run() { + signalRunLatch.trigger(); + countDownLatch.countDown(); + + try { + countDownLatch.await(); + } catch (InterruptedException e) { + ExceptionUtils.rethrow(e); + } + + future.complete(value); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + future.cancel(mayInterruptIfRunning); + return true; + } + + @Override + public boolean isCancelled() { + return future.isCancelled(); + } + + @Override + public boolean isDone() { + return future.isDone(); + } + + @Override + public SnapshotResult get() throws InterruptedException, ExecutionException { + return future.get(); + } + + @Override + public SnapshotResult get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException { + return future.get(); + } + + void awaitRun() throws InterruptedException { + signalRunLatch.await(); + } + } + + private static class CheckpointOperator implements OneInputStreamOperator { + + private static final long serialVersionUID = 1L; + + private final OperatorSnapshotFutures operatorSnapshotFutures; + + private boolean checkpointed = false; + + CheckpointOperator(OperatorSnapshotFutures operatorSnapshotFutures) { + this.operatorSnapshotFutures = operatorSnapshotFutures; + } + + boolean isCheckpointed() { + return checkpointed; + } + + @Override + public void open() throws Exception { + } + + @Override + public void close() throws Exception { + } + + @Override + public void dispose() { + } + + @Override + public void prepareSnapshotPreBarrier(long checkpointId) { + } + + @Override + public OperatorSnapshotFutures snapshotState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions, CheckpointStreamFactory storageLocation) throws Exception { + this.checkpointed = true; + return operatorSnapshotFutures; + } + + @Override + public void initializeState(StreamTaskStateInitializer streamTaskStateManager) throws Exception { + } + + @Override + public void setKeyContextElement1(StreamRecord record) { + } + + @Override + public void setKeyContextElement2(StreamRecord record) { + } + + @Override + public MetricGroup getMetricGroup() { + return null; + } + + @Override + public OperatorID getOperatorID() { + return new OperatorID(); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) { + } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + + @Override + public void setCurrentKey(Object key) { + } + + @Override + public Object getCurrentKey() { + return null; + } + + @Override + public void processElement(StreamRecord element) throws Exception { + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + } + + @Override + public void processLatencyMarker(LatencyMarker latencyMarker) { + } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java index ca5ee615528b6..eff8c64c5b93c 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java @@ -68,6 +68,7 @@ import org.junit.rules.Timeout; import java.util.Collections; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -174,6 +175,11 @@ public Future notifyCheckpointCompleteAsync(long checkpointId) { } } + @Override + public Future notifyCheckpointAbortAsync(long checkpointId) { + return CompletableFuture.completedFuture(null); + } + @Override protected void init() { } diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/FailingCollectionSource.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/FailingCollectionSource.java index 28d92b962333d..df3a5689f862c 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/FailingCollectionSource.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/FailingCollectionSource.java @@ -245,6 +245,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { lastCheckpointedEmittedNum = checkpointedEmittedNums.get(checkpointId); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + public static void reset() { failedBefore = false; } diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/FsStreamingSinkITCaseBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/FsStreamingSinkITCaseBase.scala index 56b7d21301403..02b70b45cadb4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/FsStreamingSinkITCaseBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/FsStreamingSinkITCaseBase.scala @@ -176,4 +176,7 @@ class FiniteTestSource(elements: Iterable[Row]) extends SourceFunction[Row] with override def notifyCheckpointComplete(checkpointId: Long): Unit = { numCheckpointsComplete += 1 } + + @throws[Exception] + override def notifyCheckpointAborted(checkpointId: Long): Unit = {} } diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java index b3a9546b3ddeb..c85aaaa06cf99 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/streaming/util/FiniteTestSource.java @@ -91,4 +91,8 @@ public void cancel() { public void notifyCheckpointComplete(long checkpointId) throws Exception { numCheckpointsComplete++; } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointIT.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointIT.java index 12d186fcbef4b..42c889c5b45b6 100644 --- a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointIT.java +++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointIT.java @@ -352,6 +352,11 @@ public Future notifyCheckpointCompleteAsync(long checkpointId) { return super.notifyCheckpointCompleteAsync(checkpointId); } + + @Override + public Future notifyCheckpointAbortAsync(long checkpointId) { + return CompletableFuture.completedFuture(null); + } } /** diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java index 419369434ba40..8605623befb94 100644 --- a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java @@ -251,6 +251,11 @@ public Future triggerCheckpointAsync(final CheckpointMetaData checkpoin public Future notifyCheckpointCompleteAsync(final long checkpointId) { return CompletableFuture.completedFuture(null); } + + @Override + public Future notifyCheckpointAbortAsync(long checkpointId) { + return CompletableFuture.completedFuture(null); + } } private String cancelWithSavepoint() throws Exception { diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CoStreamCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CoStreamCheckpointingITCase.java index b207de8c0cc27..cf49d5ae90c00 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CoStreamCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/CoStreamCheckpointingITCase.java @@ -227,6 +227,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + private static String randomString(StringBuilder bld, Random rnd) { final int len = rnd.nextInt(10) + 5; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ContinuousFileProcessingCheckpointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ContinuousFileProcessingCheckpointITCase.java index 612ed4371e783..fa6a0d155ec6c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ContinuousFileProcessingCheckpointITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ContinuousFileProcessingCheckpointITCase.java @@ -247,6 +247,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { this.successfulCheckpoints++; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + private int getFileIdx(String line) { String[] tkns = line.split(":"); return Integer.parseInt(tkns[0]); diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java index e7f992145cb21..b2895e61a81a2 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java @@ -265,6 +265,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { this.notifyAll(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } private static class OnceFailingPartitionedSum diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java index 0fcfb8fe108b5..a24384344307d 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StateCheckpointedITCase.java @@ -364,6 +364,10 @@ public void restoreState(List> state) throws Except public void notifyCheckpointComplete(long checkpointId) { this.wasCheckpointed = true; } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } private static class ValidatingSink extends RichSinkFunction diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java index 7b058a055a7c7..ece4f629a8cab 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointNotifierITCase.java @@ -250,6 +250,10 @@ public void notifyCheckpointComplete(long checkpointId) { GeneratingSourceFunction.numPostFailureNotifications.incrementAndGet(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } /** @@ -281,6 +285,10 @@ public void notifyCheckpointComplete(long checkpointId) { GeneratingSourceFunction.numPostFailureNotifications.incrementAndGet(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } /** @@ -312,6 +320,10 @@ public void notifyCheckpointComplete(long checkpointId) { GeneratingSourceFunction.numPostFailureNotifications.incrementAndGet(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } /** @@ -349,6 +361,10 @@ public void notifyCheckpointComplete(long checkpointId) { GeneratingSourceFunction.numPostFailureNotifications.incrementAndGet(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } /** @@ -414,5 +430,9 @@ public void notifyCheckpointComplete(long checkpointId) { GeneratingSourceFunction.numPostFailureNotifications.incrementAndGet(); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java index b8f3f162b3a1f..47e7007661ae4 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java @@ -247,6 +247,10 @@ public void notifyCheckpointComplete(long checkpointId) { state.numCompletedCheckpoints++; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void run(SourceContext ctx) throws Exception { int increment = getRuntimeContext().getNumberOfParallelSubtasks(); @@ -445,6 +449,10 @@ public void notifyCheckpointComplete(long checkpointId) { state.completedCheckpoints++; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { checkFail(failDuringSnapshot, "snapshotState"); diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java index 5758f76af0686..81df28f0b7e3c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java @@ -422,6 +422,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { checkpointCompletedIncludingData.compareAndSet(false, true); } } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } /** diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/AccumulatingIntegerSink.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/AccumulatingIntegerSink.java index a4b66d666f2a6..4fd967c2a1d2a 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/AccumulatingIntegerSink.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/AccumulatingIntegerSink.java @@ -75,6 +75,10 @@ public void notifyCheckpointComplete(long checkpointId) { pendingForAccumulator.remove(checkpointId).forEach(accumulator::add); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @SuppressWarnings("unchecked") public static List getOutput(Map accumulators) { return (List) accumulators.get(ACCUMULATOR_NAME); diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/CancellingIntegerSource.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/CancellingIntegerSource.java index 1f49c2ed0571b..93ad11b7c9068 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/CancellingIntegerSource.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/CancellingIntegerSource.java @@ -111,6 +111,10 @@ public void notifyCheckpointComplete(long checkpointId) { } } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void cancel() { isCanceled = true; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/FailingSource.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/FailingSource.java index ff4995996e6c2..82ca91e349165 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/FailingSource.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/FailingSource.java @@ -134,6 +134,10 @@ public void notifyCheckpointComplete(long checkpointId) { checkpointStatus.compareAndSet(checkpointId, STATEFUL_CHECKPOINT_COMPLETED); } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public List snapshotState(long checkpointId, long timestamp) throws Exception { // We accept a checkpoint as basis if it should have a "decent amount" of state diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java index 234c473ef737d..f1fae2c5c41b5 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java +++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointedStreamingProgram.java @@ -123,6 +123,10 @@ public String map(String value) throws Exception { public void notifyCheckpointComplete(long checkpointId) throws Exception { atLeastOneSnapshotComplete = true; } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + } } // -------------------------------------------------------------------------------------------- diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java index 0d5f2d7490639..ca178c7bb98bb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java +++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/CheckpointingCustomKvStateProgram.java @@ -166,6 +166,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { atLeastOneSnapshotComplete = true; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + private static class ReduceSum implements ReduceFunction { private static final long serialVersionUID = 1L; diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java index db03220d32451..29c03465724e2 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/ReinterpretDataStreamAsKeyedStreamITCase.java @@ -256,6 +256,10 @@ public void notifyCheckpointComplete(long checkpointId) { canFail = !isRestored; } + @Override + public void notifyCheckpointAborted(long checkpointId) { + } + @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { positionState.clear(); From fcacc42e17f00cb47c5c16fe75af035f784ae1fa Mon Sep 17 00:00:00 2001 From: Yun Tang Date: Mon, 11 May 2020 13:49:18 +0800 Subject: [PATCH 067/773] [FLINK-8871][checkpoint] Support to cancel checkpoing via notification on checkpoint coordinator side --- .../checkpoint/CheckpointCoordinator.java | 16 +++++++++++++ .../checkpoint/CheckpointFailureManager.java | 1 + .../checkpoint/CheckpointFailureReason.java | 2 ++ .../runtime/executiongraph/Execution.java | 19 +++++++++++++++ .../jobmanager/slots/TaskManagerGateway.java | 14 +++++++++++ .../jobmaster/RpcTaskManagerGateway.java | 5 ++++ .../runtime/taskexecutor/TaskExecutor.java | 23 ++++++++++++++++++- .../taskexecutor/TaskExecutorGateway.java | 11 +++++++++ .../checkpoint/CheckpointCoordinatorTest.java | 7 ++++++ .../utils/SimpleAckingTaskManagerGateway.java | 7 ++++++ .../TestingTaskExecutorGateway.java | 5 ++++ 11 files changed, 109 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java index 9e2a4a74489da..7689f29c7a5b4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java @@ -113,6 +113,7 @@ public class CheckpointCoordinator { private final ExecutionVertex[] tasksToWaitFor; /** Tasks who need to be sent a message when a checkpoint is confirmed. */ + // TODO currently we use commit vertices to receive "abort checkpoint" messages. private final ExecutionVertex[] tasksToCommitTo; /** The operator coordinators that need to be checkpointed. */ @@ -1015,6 +1016,7 @@ public void run() { } }); + sendAbortedMessages(checkpointId, pendingCheckpoint.getCheckpointTimestamp()); throw new CheckpointException("Could not complete the pending checkpoint " + checkpointId + '.', CheckpointFailureReason.FINALIZE_CHECKPOINT_FAILURE, exception); } @@ -1067,6 +1069,19 @@ private void sendAcknowledgeMessages(long checkpointId, long timestamp) { } } + private void sendAbortedMessages(long checkpointId, long timeStamp) { + // send notification of aborted checkpoints asynchronously. + executor.execute(() -> { + // send the "abort checkpoint" messages to necessary vertices. + for (ExecutionVertex ev : tasksToCommitTo) { + Execution ee = ev.getCurrentExecutionAttempt(); + if (ee != null) { + ee.notifyCheckpointAborted(checkpointId, timeStamp); + } + } + }); + } + /** * Fails all pending checkpoints which have not been acknowledged by the given execution * attempt id. @@ -1576,6 +1591,7 @@ private void abortPendingCheckpoint( exception, pendingCheckpoint.getCheckpointId()); } } finally { + sendAbortedMessages(pendingCheckpoint.getCheckpointId(), pendingCheckpoint.getCheckpointTimestamp()); pendingCheckpoints.remove(pendingCheckpoint.getCheckpointId()); rememberRecentCheckpointId(pendingCheckpoint.getCheckpointId()); timer.execute(this::executeQueuedRequest); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureManager.java index 9e162e74f20f1..0dc655bda5b47 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureManager.java @@ -120,6 +120,7 @@ public void checkFailureCounter( case CHECKPOINT_EXPIRED: case TASK_FAILURE: case TASK_CHECKPOINT_FAILURE: + case UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE: case TRIGGER_CHECKPOINT_FAILURE: case FINALIZE_CHECKPOINT_FAILURE: //ignore diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureReason.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureReason.java index 023f9bf2cce27..cd787d0b472b4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureReason.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointFailureReason.java @@ -68,6 +68,8 @@ public enum CheckpointFailureReason { TASK_CHECKPOINT_FAILURE(false, "Task local checkpoint failure."), + UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE(false, "Unknown task for the checkpoint to notify."), + FINALIZE_CHECKPOINT_FAILURE(false, "Failure to finalize checkpoint."), TRIGGER_CHECKPOINT_FAILURE(false, "Trigger checkpoint failure."); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java index 7d60a6c548b12..0102ce702f6ff 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java @@ -993,6 +993,25 @@ public void notifyCheckpointComplete(long checkpointId, long timestamp) { } } + /** + * Notify the task of this execution about a aborted checkpoint. + * + * @param abortCheckpointId of the subsumed checkpoint + * @param timestamp of the subsumed checkpoint + */ + public void notifyCheckpointAborted(long abortCheckpointId, long timestamp) { + final LogicalSlot slot = assignedResource; + + if (slot != null) { + final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway(); + + taskManagerGateway.notifyCheckpointAborted(attemptId, getVertex().getJobId(), abortCheckpointId, timestamp); + } else { + LOG.debug("The execution has no slot assigned. This indicates that the execution is " + + "no longer running."); + } + } + /** * Trigger a new checkpoint on the task of this execution. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java index da21982a171f4..e7cbeae11b1a3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java @@ -119,6 +119,20 @@ void notifyCheckpointComplete( long checkpointId, long timestamp); + /** + * Notify the given task about a aborted checkpoint. + * + * @param executionAttemptID identifying the task + * @param jobId identifying the job to which the task belongs + * @param checkpointId of the subsumed checkpoint + * @param timestamp of the subsumed checkpoint + */ + void notifyCheckpointAborted( + ExecutionAttemptID executionAttemptID, + JobID jobId, + long checkpointId, + long timestamp); + /** * Trigger for the given task a checkpoint. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java index 9aec97e80f622..2a20b71d6bd1a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java @@ -92,6 +92,11 @@ public void notifyCheckpointComplete(ExecutionAttemptID executionAttemptID, JobI taskExecutorGateway.confirmCheckpoint(executionAttemptID, checkpointId, timestamp); } + @Override + public void notifyCheckpointAborted(ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long timestamp) { + taskExecutorGateway.abortCheckpoint(executionAttemptID, checkpointId, timestamp); + } + @Override public void triggerCheckpoint(ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long timestamp, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) { taskExecutorGateway.triggerCheckpoint( diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index d74532ff16799..f8438f5910888 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -860,7 +860,28 @@ public CompletableFuture confirmCheckpoint( final String message = "TaskManager received a checkpoint confirmation for unknown task " + executionAttemptID + '.'; log.debug(message); - return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.TASK_CHECKPOINT_FAILURE)); + return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE)); + } + } + + @Override + public CompletableFuture abortCheckpoint( + ExecutionAttemptID executionAttemptID, + long checkpointId, + long checkpointTimestamp) { + log.debug("Abort checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID); + + final Task task = taskSlotTable.getTask(executionAttemptID); + + if (task != null) { + task.notifyCheckpointAborted(checkpointId); + + return CompletableFuture.completedFuture(Acknowledge.get()); + } else { + final String message = "TaskManager received an aborted checkpoint for unknown task " + executionAttemptID + '.'; + + log.debug(message); + return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE)); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java index ad37db04ca7f1..6867451647a56 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java @@ -155,6 +155,17 @@ CompletableFuture triggerCheckpoint( */ CompletableFuture confirmCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp); + /** + * Abort a checkpoint for the given task. The checkpoint is identified by the checkpoint ID + * and the checkpoint timestamp. + * + * @param executionAttemptID identifying the task + * @param checkpointId unique id for the checkpoint + * @param checkpointTimestamp is the timestamp when the checkpoint has been initiated + * @return Future acknowledge if the checkpoint has been successfully confirmed + */ + CompletableFuture abortCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp); + /** * Cancel the given task. * diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java index ac48106d697b3..e79983648a58f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java @@ -425,6 +425,9 @@ public void testTriggerAndDeclineCheckpointComplex() { // decline checkpoint from one of the tasks, this should cancel the checkpoint coord.receiveDeclineMessage(new DeclineCheckpoint(jid, attemptID1, checkpoint1Id), TASK_MANAGER_LOCATION_INFO); + verify(vertex1.getCurrentExecutionAttempt(), times(1)).notifyCheckpointAborted(eq(checkpoint1Id), any(Long.class)); + verify(vertex2.getCurrentExecutionAttempt(), times(1)).notifyCheckpointAborted(eq(checkpoint1Id), any(Long.class)); + assertTrue(checkpoint1.isDiscarded()); // validate that we have only one pending checkpoint left @@ -453,6 +456,10 @@ public void testTriggerAndDeclineCheckpointComplex() { coord.receiveDeclineMessage(new DeclineCheckpoint(jid, attemptID2, checkpoint1Id), TASK_MANAGER_LOCATION_INFO); assertTrue(checkpoint1.isDiscarded()); + // will not notify abort message again + verify(vertex1.getCurrentExecutionAttempt(), times(1)).notifyCheckpointAborted(eq(checkpoint1Id), any(Long.class)); + verify(vertex2.getCurrentExecutionAttempt(), times(1)).notifyCheckpointAborted(eq(checkpoint1Id), any(Long.class)); + coord.shutdown(JobStatus.FINISHED); } catch (Exception e) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java index 7e6ee2d8b3c42..59aee819f9573 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java @@ -128,6 +128,13 @@ public void notifyCheckpointComplete( long checkpointId, long timestamp) {} + @Override + public void notifyCheckpointAborted( + ExecutionAttemptID executionAttemptID, + JobID jobId, + long checkpointId, + long timestamp) {} + @Override public void triggerCheckpoint( ExecutionAttemptID executionAttemptID, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java index f7e82d2858062..2a039c3c4b00f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java @@ -168,6 +168,11 @@ public CompletableFuture confirmCheckpoint(ExecutionAttemptID execu return CompletableFuture.completedFuture(Acknowledge.get()); } + @Override + public CompletableFuture abortCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) { + return CompletableFuture.completedFuture(Acknowledge.get()); + } + @Override public CompletableFuture cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) { return cancelTaskFunction.apply(executionAttemptID); From bbf731869adeaf7d7183d6d7299a0d1406afd1bc Mon Sep 17 00:00:00 2001 From: Yun Tang Date: Sun, 17 May 2020 21:30:30 +0800 Subject: [PATCH 068/773] [FLINK-8871][checkpoint][tests] Add ITcase for NotifiCheckpointAborted mechanism --- .../tasks/ExceptionallyDoneFuture.java | 2 +- .../NotifyCheckpointAbortedITCase.java | 449 ++++++++++++++++++ 2 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/NotifyCheckpointAbortedITCase.java diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/ExceptionallyDoneFuture.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/ExceptionallyDoneFuture.java index 55bfc18b59a25..f95ec5c0c5204 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/ExceptionallyDoneFuture.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/ExceptionallyDoneFuture.java @@ -27,7 +27,7 @@ * * @param type of the RunnableFuture */ -class ExceptionallyDoneFuture implements RunnableFuture { +public class ExceptionallyDoneFuture implements RunnableFuture { private final Throwable throwable; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/NotifyCheckpointAbortedITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/NotifyCheckpointAbortedITCase.java new file mode 100644 index 0000000000000..e7e6e5ca68c8b --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/NotifyCheckpointAbortedITCase.java @@ -0,0 +1,449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.checkpointing; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.client.ClientUtils; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.HighAvailabilityOptions; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.core.fs.CloseableRegistry; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.testutils.OneShotLatch; +import org.apache.flink.runtime.checkpoint.CheckpointOptions; +import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory; +import org.apache.flink.runtime.checkpoint.CompletedCheckpoint; +import org.apache.flink.runtime.checkpoint.StandaloneCheckpointIDCounter; +import org.apache.flink.runtime.checkpoint.StandaloneCompletedCheckpointStore; +import org.apache.flink.runtime.checkpoint.TestingCheckpointRecoveryFactory; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; +import org.apache.flink.runtime.highavailability.HighAvailabilityServicesFactory; +import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.operators.testutils.ExpectedTestException; +import org.apache.flink.runtime.state.AbstractSnapshotStrategy; +import org.apache.flink.runtime.state.BackendBuildingException; +import org.apache.flink.runtime.state.CheckpointStreamFactory; +import org.apache.flink.runtime.state.DefaultOperatorStateBackend; +import org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder; +import org.apache.flink.runtime.state.DoneFuture; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.runtime.state.FunctionSnapshotContext; +import org.apache.flink.runtime.state.OperatorStateBackend; +import org.apache.flink.runtime.state.OperatorStateHandle; +import org.apache.flink.runtime.state.SnapshotResult; +import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.streaming.api.CheckpointingMode; +import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.api.operators.StreamMap; +import org.apache.flink.streaming.api.operators.StreamSink; +import org.apache.flink.streaming.runtime.tasks.ExceptionallyDoneFuture; +import org.apache.flink.test.util.MiniClusterWithClientResource; +import org.apache.flink.util.TestLogger; + +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import javax.annotation.Nonnull; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; + +/** + * Integrated tests to verify the logic to notify checkpoint aborted via RPC message. + */ +@RunWith(Parameterized.class) +public class NotifyCheckpointAbortedITCase extends TestLogger { + private static final long DECLINE_CHECKPOINT_ID = 2L; + private static final long TEST_TIMEOUT = 60000; + private static final String DECLINE_SINK_NAME = "DeclineSink"; + private static MiniClusterWithClientResource cluster; + + private static Path checkpointPath; + + @Parameterized.Parameter + public boolean unalignedCheckpointEnabled; + + @Parameterized.Parameters(name = "unalignedCheckpointEnabled ={0}") + public static Collection parameter() { + return Arrays.asList(true, false); + } + + @ClassRule + public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); + + @Before + public void setup() throws Exception { + Configuration configuration = new Configuration(); + configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true); + configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName()); + + checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI()); + cluster = new MiniClusterWithClientResource( + new MiniClusterResourceConfiguration.Builder() + .setConfiguration(configuration) + .setNumberTaskManagers(1) + .setNumberSlotsPerTaskManager(1).build()); + cluster.before(); + + NormalMap.reset(); + DeclineSink.reset(); + TestingCompletedCheckpointStore.reset(); + } + + @After + public void shutdown() { + if (cluster != null) { + cluster.after(); + cluster = null; + } + + } + + /** + * Verify operators would be notified as checkpoint aborted. + * + *

The job would run with at least two checkpoints. The 1st checkpoint would fail due to add checkpoint to store, + * and the 2nd checkpoint would decline by async checkpoint phase of 'DeclineSink'. + * + *

The job graph looks like: + * NormalSource --> keyBy --> NormalMap --> DeclineSink + */ + @Test(timeout = TEST_TIMEOUT) + public void testNotifyCheckpointAborted() throws Exception { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.enableCheckpointing(200, CheckpointingMode.EXACTLY_ONCE); + env.getCheckpointConfig().enableUnalignedCheckpoints(unalignedCheckpointEnabled); + env.getCheckpointConfig().setTolerableCheckpointFailureNumber(1); + env.disableOperatorChaining(); + env.setParallelism(1); + + final StateBackend failingStateBackend = new DeclineSinkFailingStateBackend(checkpointPath); + env.setStateBackend(failingStateBackend); + + env.addSource(new NormalSource()).name("NormalSource") + .keyBy((KeySelector, Integer>) value -> value.f0) + .transform("NormalMap", TypeInformation.of(Integer.class), new NormalMap()) + .transform(DECLINE_SINK_NAME, TypeInformation.of(Object.class), new DeclineSink()); + + final ClusterClient clusterClient = cluster.getClusterClient(); + JobGraph jobGraph = env.getStreamGraph().getJobGraph(); + JobID jobID = jobGraph.getJobID(); + + ClientUtils.submitJob(clusterClient, jobGraph); + + TestingCompletedCheckpointStore.addCheckpointLatch.await(); + TestingCompletedCheckpointStore.abortCheckpointLatch.trigger(); + + verifyAllOperatorsNotifyAborted(); + resetAllOperatorsNotifyAbortedLatches(); + verifyAllOperatorsNotifyAbortedTimes(1); + + DeclineSink.waitLatch.trigger(); + verifyAllOperatorsNotifyAborted(); + verifyAllOperatorsNotifyAbortedTimes(2); + + clusterClient.cancel(jobID).get(); + } + + private void verifyAllOperatorsNotifyAborted() throws InterruptedException { + NormalMap.notifiedAbortedLatch.await(); + DeclineSink.notifiedAbortedLatch.await(); + } + + private void resetAllOperatorsNotifyAbortedLatches() { + NormalMap.notifiedAbortedLatch.reset(); + DeclineSink.notifiedAbortedLatch.reset(); + } + + private void verifyAllOperatorsNotifyAbortedTimes(int expectedTimes) { + assertEquals(expectedTimes, NormalMap.notifiedAbortedTimes.get()); + assertEquals(expectedTimes, DeclineSink.notifiedAbortedTimes.get()); + } + + /** + * Normal source function. + */ + private static class NormalSource implements SourceFunction> { + private static final long serialVersionUID = 1L; + protected volatile boolean running; + + NormalSource() { + this.running = true; + } + + @Override + public void run(SourceContext> ctx) throws Exception { + while (running) { + synchronized (ctx.getCheckpointLock()) { + ctx.collect(Tuple2.of(ThreadLocalRandom.current().nextInt(), ThreadLocalRandom.current().nextInt())); + } + Thread.sleep(10); + } + } + + @Override + public void cancel() { + this.running = false; + } + } + + private static class NormalMap extends StreamMap, Integer> { + private static final long serialVersionUID = 1L; + private static final OneShotLatch notifiedAbortedLatch = new OneShotLatch(); + private static final AtomicInteger notifiedAbortedTimes = new AtomicInteger(0); + + public NormalMap() { + super(new NormalMapFunction()); + } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + notifiedAbortedTimes.incrementAndGet(); + notifiedAbortedLatch.trigger(); + } + + static void reset() { + notifiedAbortedLatch.reset(); + notifiedAbortedTimes.set(0); + } + } + + /** + * Normal map function. + */ + private static class NormalMapFunction implements MapFunction, Integer>, CheckpointedFunction { + private static final long serialVersionUID = 1L; + private ValueState valueState; + + @Override + public Integer map(Tuple2 value) throws Exception { + valueState.update(value.f1); + return value.f1; + } + + @Override + public void snapshotState(FunctionSnapshotContext context) { + } + + @Override + public void initializeState(FunctionInitializationContext context) throws Exception { + valueState = context.getKeyedStateStore().getState(new ValueStateDescriptor<>("value", Integer.class)); + } + } + + /** + * A decline sink. + */ + private static class DeclineSink extends StreamSink { + private static final long serialVersionUID = 1L; + private static final OneShotLatch notifiedAbortedLatch = new OneShotLatch(); + private static final OneShotLatch waitLatch = new OneShotLatch(); + private static final AtomicInteger notifiedAbortedTimes = new AtomicInteger(0); + + public DeclineSink() { + super(new SinkFunction() { + private static final long serialVersionUID = 1L; + }); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + if (context.getCheckpointId() == DECLINE_CHECKPOINT_ID) { + DeclineSink.waitLatch.await(); + } + super.snapshotState(context); + } + + @Override + public void notifyCheckpointAborted(long checkpointId) { + notifiedAbortedTimes.incrementAndGet(); + notifiedAbortedLatch.trigger(); + } + + static void reset() { + notifiedAbortedLatch.reset(); + waitLatch.reset(); + notifiedAbortedTimes.set(0); + } + + } + + /** + * The snapshot strategy to create failing runnable future at the checkpoint to decline. + */ + private static class DeclineSinkFailingSnapshotStrategy extends AbstractSnapshotStrategy { + + protected DeclineSinkFailingSnapshotStrategy() { + super("StuckAsyncSnapshotStrategy"); + } + + @Override + public RunnableFuture> snapshot( + long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) { + if (checkpointId == DECLINE_CHECKPOINT_ID) { + return ExceptionallyDoneFuture.of(new ExpectedTestException()); + } else { + return DoneFuture.of(SnapshotResult.empty()); + } + } + } + + /** + * The operator statebackend to create {@link DeclineSinkFailingSnapshotStrategy} at {@link DeclineSink}. + */ + private static class DeclineSinkFailingOperatorStateBackend extends DefaultOperatorStateBackend { + + public DeclineSinkFailingOperatorStateBackend( + ExecutionConfig executionConfig, + CloseableRegistry closeStreamOnCancelRegistry, + AbstractSnapshotStrategy snapshotStrategy) { + super(executionConfig, + closeStreamOnCancelRegistry, + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + snapshotStrategy); + } + } + + /** + * The state backend to create {@link DeclineSinkFailingOperatorStateBackend} at {@link DeclineSink}. + */ + private static class DeclineSinkFailingStateBackend extends FsStateBackend { + private static final long serialVersionUID = 1L; + + public DeclineSinkFailingStateBackend(Path checkpointDataUri) { + super(checkpointDataUri); + } + + @Override + public DeclineSinkFailingStateBackend configure(ReadableConfig config, ClassLoader classLoader) { + return new DeclineSinkFailingStateBackend(checkpointPath); + } + + @Override + public OperatorStateBackend createOperatorStateBackend( + Environment env, + String operatorIdentifier, + @Nonnull Collection stateHandles, + CloseableRegistry cancelStreamRegistry) throws BackendBuildingException { + if (operatorIdentifier.contains(DECLINE_SINK_NAME)) { + return new DeclineSinkFailingOperatorStateBackend( + env.getExecutionConfig(), + cancelStreamRegistry, + new DeclineSinkFailingSnapshotStrategy()); + } else { + return new DefaultOperatorStateBackendBuilder( + env.getUserClassLoader(), + env.getExecutionConfig(), + false, + stateHandles, + cancelStreamRegistry).build(); + } + } + } + + private static class TestingHaServices extends EmbeddedHaServices { + private final CheckpointRecoveryFactory checkpointRecoveryFactory; + + TestingHaServices(CheckpointRecoveryFactory checkpointRecoveryFactory, Executor executor) { + super(executor); + this.checkpointRecoveryFactory = checkpointRecoveryFactory; + } + + @Override + public CheckpointRecoveryFactory getCheckpointRecoveryFactory() { + return checkpointRecoveryFactory; + } + } + + /** + * An extension of {@link StandaloneCompletedCheckpointStore}. + */ + private static class TestingCompletedCheckpointStore extends StandaloneCompletedCheckpointStore { + private static final OneShotLatch addCheckpointLatch = new OneShotLatch(); + private static final OneShotLatch abortCheckpointLatch = new OneShotLatch(); + + TestingCompletedCheckpointStore() { + super(1); + } + + @Override + public void addCheckpoint(CompletedCheckpoint checkpoint) throws Exception { + if (abortCheckpointLatch.isTriggered()) { + super.addCheckpoint(checkpoint); + } else { + // tell main thread that all checkpoints on task side have been finished. + addCheckpointLatch.trigger(); + // wait for the main thread to throw exception so that the checkpoint would be notified as aborted. + abortCheckpointLatch.await(); + throw new ExpectedTestException(); + } + } + + static void reset() { + addCheckpointLatch.reset(); + abortCheckpointLatch.reset(); + } + } + + /** + * Testing HA factory which needs to be public in order to be instantiatable. + */ + public static class TestingHAFactory implements HighAvailabilityServicesFactory { + + @Override + public HighAvailabilityServices createHAServices(Configuration configuration, Executor executor) { + return new TestingHaServices( + new TestingCheckpointRecoveryFactory(new TestingCompletedCheckpointStore(), new StandaloneCheckpointIDCounter()), + executor); + } + } + +} From c3ff1de47cd01d7448d325c42d9ad76681e8c85d Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 18 May 2020 11:07:22 +0200 Subject: [PATCH 069/773] [hotfix][table] Reduce friction around logical type roots --- .../table/types/logical/LogicalTypeRoot.java | 10 + .../logical/utils/LogicalTypeChecks.java | 13 + .../types/logical/utils/LogicalTypeUtils.java | 38 +- .../table/planner/codegen/CodeGenUtils.scala | 499 ++++++++++-------- .../codegen/EqualiserCodeGenerator.scala | 17 +- .../planner/codegen/ExpressionReducer.scala | 6 +- .../table/planner/codegen/GenerateUtils.scala | 237 ++++++--- .../codegen/agg/batch/AggCodeGenHelper.scala | 34 +- .../runtime/typeutils/TypeCheckUtils.java | 18 +- 9 files changed, 531 insertions(+), 341 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java index 0079a8de042ac..e2a97f678d23f 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java @@ -37,6 +37,16 @@ * {@code SYMBOL}, or {@code RAW}). * *

See the type-implementing classes for a more detailed description of each type. + * + *

Note to implementers: Whenever we perform a match against a type root (e.g. using a switch/case + * statement), it is recommended to: + *

    + *
  • Order the items by the type root definition in this class for easy readability. + *
  • Think about the behavior of all type roots for the implementation. A default fallback is + * dangerous when introducing a new type root in the future. + *
  • In many runtime cases, resolve the indirection of {@link #DISTINCT_TYPE}: + * {@code return myMethod(((DistinctType) type).getSourceType)} + *
*/ @PublicEvolving public enum LogicalTypeRoot { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeChecks.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeChecks.java index 8a3e301930ba3..b6117f45fe114 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeChecks.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeChecks.java @@ -108,6 +108,9 @@ public static boolean isProctimeAttribute(LogicalType logicalType) { /** * Checks if the given type is a composite type. * + *

Use {@link #getFieldCount(LogicalType)}, {@link #getFieldNames(LogicalType)}, + * {@link #getFieldTypes(LogicalType)} for unified handling of composite types. + * * @param logicalType Logical data type to check * @return True if the type is composite type. */ @@ -198,6 +201,16 @@ public static List getFieldNames(LogicalType logicalType) { return logicalType.accept(FIELD_NAMES_EXTRACTOR); } + /** + * Returns the field types of row and structured types. + */ + public static List getFieldTypes(LogicalType logicalType) { + if (logicalType instanceof DistinctType) { + return getFieldTypes(((DistinctType) logicalType).getSourceType()); + } + return logicalType.getChildren(); + } + private LogicalTypeChecks() { // no instantiation } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java index 5e8be860d4100..033d71125264b 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java @@ -26,6 +26,7 @@ import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.TimestampType; @@ -45,14 +46,23 @@ public static LogicalType removeTimeAttributes(LogicalType logicalType) { /** * Returns the conversion class for the given {@link LogicalType} that is used by the - * table runtime. + * table runtime as internal data structure. * * @see RowData */ public static Class toInternalConversionClass(LogicalType type) { + // ordered by type root definition switch (type.getTypeRoot()) { + case CHAR: + case VARCHAR: + return StringData.class; case BOOLEAN: return Boolean.class; + case BINARY: + case VARBINARY: + return byte[].class; + case DECIMAL: + return DecimalData.class; case TINYINT: return Byte.class; case SMALLINT: @@ -65,32 +75,32 @@ public static Class toInternalConversionClass(LogicalType type) { case BIGINT: case INTERVAL_DAY_TIME: return Long.class; - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return TimestampData.class; case FLOAT: return Float.class; case DOUBLE: return Double.class; - case CHAR: - case VARCHAR: - return StringData.class; - case DECIMAL: - return DecimalData.class; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return TimestampData.class; + case TIMESTAMP_WITH_TIME_ZONE: + throw new UnsupportedOperationException("Unsupported type: " + type); case ARRAY: return ArrayData.class; - case MAP: case MULTISET: + case MAP: return MapData.class; case ROW: + case STRUCTURED_TYPE: return RowData.class; - case BINARY: - case VARBINARY: - return byte[].class; + case DISTINCT_TYPE: + return toInternalConversionClass(((DistinctType) type).getSourceType()); case RAW: return RawValueData.class; + case NULL: + case SYMBOL: + case UNRESOLVED: default: - throw new UnsupportedOperationException("Unsupported type: " + type); + throw new IllegalArgumentException("Illegal type: " + type); } } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala index 58b70104b1990..6e62a3f4136d5 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala @@ -40,10 +40,12 @@ import org.apache.flink.table.runtime.util.MurmurHashUtil import org.apache.flink.table.types.DataType import org.apache.flink.table.types.logical.LogicalTypeRoot._ import org.apache.flink.table.types.logical._ -import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.hasRoot +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{getFieldCount, getPrecision, getScale, hasRoot} import org.apache.flink.table.types.logical.utils.LogicalTypeUtils.toInternalConversionClass import org.apache.flink.types.{Row, RowKind} +import scala.annotation.tailrec + object CodeGenUtils { // ------------------------------- DEFAULT TERMS ------------------------------------------ @@ -161,117 +163,118 @@ object CodeGenUtils { // works, but for boxed types we need this: // Float a = 1.0f; // Byte b = (byte)(float) a; + @tailrec def primitiveTypeTermForType(t: LogicalType): String = t.getTypeRoot match { - case INTEGER => "int" - case BIGINT => "long" - case SMALLINT => "short" + // ordered by type root definition + case BOOLEAN => "boolean" case TINYINT => "byte" + case SMALLINT => "short" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => "int" + case BIGINT | INTERVAL_DAY_TIME => "long" case FLOAT => "float" case DOUBLE => "double" - case BOOLEAN => "boolean" - - case DATE => "int" - case TIME_WITHOUT_TIME_ZONE => "int" - case INTERVAL_YEAR_MONTH => "int" - case INTERVAL_DAY_TIME => "long" - + case DISTINCT_TYPE => primitiveTypeTermForType(t.asInstanceOf[DistinctType].getSourceType) case _ => boxedTypeTermForType(t) } + @tailrec def boxedTypeTermForType(t: LogicalType): String = t.getTypeRoot match { - case INTEGER => className[JInt] - case BIGINT => className[JLong] - case SMALLINT => className[JShort] + // ordered by type root definition + case CHAR | VARCHAR => BINARY_STRING + case BOOLEAN => className[JBoolean] + case BINARY | VARBINARY => "byte[]" + case DECIMAL => className[DecimalData] case TINYINT => className[JByte] + case SMALLINT => className[JShort] + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => className[JInt] + case BIGINT | INTERVAL_DAY_TIME => className[JLong] case FLOAT => className[JFloat] case DOUBLE => className[JDouble] - case BOOLEAN => className[JBoolean] - - case DATE => className[JInt] - case TIME_WITHOUT_TIME_ZONE => className[JInt] - case INTERVAL_YEAR_MONTH => className[JInt] - case INTERVAL_DAY_TIME => className[JLong] - - case VARCHAR | CHAR => BINARY_STRING - case VARBINARY | BINARY => "byte[]" - - case DECIMAL => className[DecimalData] + case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => className[TimestampData] + case TIMESTAMP_WITH_TIME_ZONE => + throw new UnsupportedOperationException("Unsupported type: " + t) case ARRAY => className[ArrayData] case MULTISET | MAP => className[MapData] - case ROW => className[RowData] + case ROW | STRUCTURED_TYPE => className[RowData] case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => className[TimestampData] - + case DISTINCT_TYPE => boxedTypeTermForType(t.asInstanceOf[DistinctType].getSourceType) + case NULL => className[JObject] // special case for untyped null literals case RAW => className[BinaryRawValueData[_]] - - // special case for untyped null literals - case NULL => className[JObject] + case SYMBOL | UNRESOLVED => + throw new IllegalArgumentException("Illegal type: " + t) } /** * Gets the default value for a primitive type, and null for generic types */ + @tailrec def primitiveDefaultValue(t: LogicalType): String = t.getTypeRoot match { - case INTEGER | TINYINT | SMALLINT => "-1" - case BIGINT => "-1L" + // ordered by type root definition + case CHAR | VARCHAR => s"$BINARY_STRING.EMPTY_UTF8" + case BOOLEAN => "false" + case TINYINT | SMALLINT | INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => "-1" + case BIGINT | INTERVAL_DAY_TIME => "-1L" case FLOAT => "-1.0f" case DOUBLE => "-1.0d" - case BOOLEAN => "false" - case VARCHAR | CHAR => s"$BINARY_STRING.EMPTY_UTF8" - case DATE | TIME_WITHOUT_TIME_ZONE => "-1" - case INTERVAL_YEAR_MONTH => "-1" - case INTERVAL_DAY_TIME => "-1L" + case DISTINCT_TYPE => primitiveDefaultValue(t.asInstanceOf[DistinctType].getSourceType) case _ => "null" } - /** - * If it's internally compatible, don't need to DataStructure converter. - * clazz != classOf[Row] => Row can only infer GenericType[Row]. - */ - def isInternalClass(t: DataType): Boolean = { - val clazz = t.getConversionClass - clazz != classOf[Object] && clazz != classOf[Row] && - (classOf[RowData].isAssignableFrom(clazz) || - clazz == toInternalConversionClass(fromDataTypeToLogicalType(t))) - } - + @tailrec def hashCodeForType( - ctx: CodeGeneratorContext, t: LogicalType, term: String): String = t.getTypeRoot match { - case BOOLEAN => s"${className[JBoolean]}.hashCode($term)" - case TINYINT => s"${className[JByte]}.hashCode($term)" - case SMALLINT => s"${className[JShort]}.hashCode($term)" - case INTEGER => s"${className[JInt]}.hashCode($term)" - case BIGINT => s"${className[JLong]}.hashCode($term)" + ctx: CodeGeneratorContext, + t: LogicalType, + term: String) + : String = t.getTypeRoot match { + // ordered by type root definition + case VARCHAR | CHAR => + s"$term.hashCode()" + case BOOLEAN => + s"${className[JBoolean]}.hashCode($term)" + case BINARY | VARBINARY => + s"${className[MurmurHashUtil]}.hashUnsafeBytes($term, $BYTE_ARRAY_BASE_OFFSET, $term.length)" + case DECIMAL => + s"$term.hashCode()" + case TINYINT => + s"${className[JByte]}.hashCode($term)" + case SMALLINT => + s"${className[JShort]}.hashCode($term)" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => + s"${className[JInt]}.hashCode($term)" + case BIGINT | INTERVAL_DAY_TIME => s"${className[JLong]}.hashCode($term)" case FLOAT => s"${className[JFloat]}.hashCode($term)" case DOUBLE => s"${className[JDouble]}.hashCode($term)" - case VARCHAR | CHAR => s"$term.hashCode()" - case VARBINARY | BINARY => s"${className[MurmurHashUtil]}.hashUnsafeBytes(" + - s"$term, $BYTE_ARRAY_BASE_OFFSET, $term.length)" - case DECIMAL => s"$term.hashCode()" - case DATE => s"${className[JInt]}.hashCode($term)" - case TIME_WITHOUT_TIME_ZONE => s"${className[JInt]}.hashCode($term)" case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => s"$term.hashCode()" - case INTERVAL_YEAR_MONTH => s"${className[JInt]}.hashCode($term)" + case TIMESTAMP_WITH_TIME_ZONE | ARRAY | MULTISET | MAP => + throw new UnsupportedOperationException("Unsupported type: " + t) case INTERVAL_DAY_TIME => s"${className[JLong]}.hashCode($term)" - case ARRAY => throw new IllegalArgumentException(s"Not support type to hash: $t") - case ROW => - val rowType = t.asInstanceOf[RowType] + case ROW | STRUCTURED_TYPE => + val fieldCount = getFieldCount(t) val subCtx = CodeGeneratorContext(ctx.tableConfig) val genHash = HashCodeGenerator.generateRowHash( - subCtx, rowType, "SubHashRow", (0 until rowType.getFieldCount).toArray) + subCtx, t, "SubHashRow", (0 until fieldCount).toArray) ctx.addReusableInnerClass(genHash.getClassName, genHash.getCode) val refs = ctx.addReusableObject(subCtx.references.toArray, "subRefs") val hashFunc = newName("hashFunc") ctx.addReusableMember(s"${classOf[HashFunction].getCanonicalName} $hashFunc;") ctx.addReusableInitStatement(s"$hashFunc = new ${genHash.getClassName}($refs);") s"$hashFunc.hashCode($term)" + case DISTINCT_TYPE => + hashCodeForType(ctx, t.asInstanceOf[DistinctType].getSourceType, term) case RAW => - val gt = t.asInstanceOf[TypeInformationRawType[_]] - val serTerm = ctx.addReusableObject( - gt.getTypeInformation.createSerializer(new ExecutionConfig), "serializer") + val serializer = t match { + case rt: RawType[_] => + rt.getTypeSerializer + case tirt: TypeInformationRawType[_] => + tirt.getTypeInformation.createSerializer(new ExecutionConfig) + } + val serTerm = ctx.addReusableObject(serializer, "serializer") s"$BINARY_RAW_VALUE.getJavaObjectFromRawValueData($term, $serTerm).hashCode()" + case NULL | SYMBOL | UNRESOLVED => + throw new IllegalArgumentException("Illegal type: " + t) } // ---------------------------------------------------------------------------------------------- @@ -406,6 +409,11 @@ object CodeGenUtils { throw new CodeGenException("Integer expression type expected.") } + def udfFieldName(udf: UserDefinedFunction): String = s"function_${udf.functionIdentifier}" + + def genLogInfo(logTerm: String, format: String, argTerm: String): String = + s"""$logTerm.info("$format", $argTerm);""" + // -------------------------------------------------------------------------------- // DataFormat Operations // -------------------------------------------------------------------------------- @@ -419,44 +427,50 @@ object CodeGenUtils { fieldType: LogicalType) : String = rowFieldReadAccess(ctx, index.toString, rowTerm, fieldType) + @tailrec def rowFieldReadAccess( ctx: CodeGeneratorContext, indexTerm: String, rowTerm: String, - t: LogicalType) : String = - t.getTypeRoot match { - // primitive types - case BOOLEAN => s"$rowTerm.getBoolean($indexTerm)" - case TINYINT => s"$rowTerm.getByte($indexTerm)" - case SMALLINT => s"$rowTerm.getShort($indexTerm)" - case INTEGER => s"$rowTerm.getInt($indexTerm)" - case BIGINT => s"$rowTerm.getLong($indexTerm)" - case FLOAT => s"$rowTerm.getFloat($indexTerm)" - case DOUBLE => s"$rowTerm.getDouble($indexTerm)" - case VARCHAR | CHAR => s"(($BINARY_STRING) $rowTerm.getString($indexTerm))" - case VARBINARY | BINARY => s"$rowTerm.getBinary($indexTerm)" + t: LogicalType) + : String = t.getTypeRoot match { + // ordered by type root definition + case CHAR | VARCHAR => + s"(($BINARY_STRING) $rowTerm.getString($indexTerm))" + case BOOLEAN => + s"$rowTerm.getBoolean($indexTerm)" + case BINARY | VARBINARY => + s"$rowTerm.getBinary($indexTerm)" case DECIMAL => - val dt = t.asInstanceOf[DecimalType] - s"$rowTerm.getDecimal($indexTerm, ${dt.getPrecision}, ${dt.getScale})" - - // temporal types - case DATE => s"$rowTerm.getInt($indexTerm)" - case TIME_WITHOUT_TIME_ZONE => s"$rowTerm.getInt($indexTerm)" - case TIMESTAMP_WITHOUT_TIME_ZONE => - val dt = t.asInstanceOf[TimestampType] - s"$rowTerm.getTimestamp($indexTerm, ${dt.getPrecision})" - case TIMESTAMP_WITH_LOCAL_TIME_ZONE => - val dt = t.asInstanceOf[LocalZonedTimestampType] - s"$rowTerm.getTimestamp($indexTerm, ${dt.getPrecision})" - case INTERVAL_YEAR_MONTH => s"$rowTerm.getInt($indexTerm)" - case INTERVAL_DAY_TIME => s"$rowTerm.getLong($indexTerm)" - - // complex types - case ARRAY => s"$rowTerm.getArray($indexTerm)" - case MULTISET | MAP => s"$rowTerm.getMap($indexTerm)" - case ROW => s"$rowTerm.getRow($indexTerm, ${t.asInstanceOf[RowType].getFieldCount})" - - case RAW => s"(($BINARY_RAW_VALUE) $rowTerm.getRawValue($indexTerm))" + s"$rowTerm.getDecimal($indexTerm, ${getPrecision(t)}, ${getScale(t)})" + case TINYINT => + s"$rowTerm.getByte($indexTerm)" + case SMALLINT => + s"$rowTerm.getShort($indexTerm)" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => + s"$rowTerm.getInt($indexTerm)" + case BIGINT | INTERVAL_DAY_TIME => + s"$rowTerm.getLong($indexTerm)" + case FLOAT => + s"$rowTerm.getFloat($indexTerm)" + case DOUBLE => + s"$rowTerm.getDouble($indexTerm)" + case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => + s"$rowTerm.getTimestamp($indexTerm, ${getPrecision(t)})" + case TIMESTAMP_WITH_TIME_ZONE => + throw new UnsupportedOperationException("Unsupported type: " + t) + case ARRAY => + s"$rowTerm.getArray($indexTerm)" + case MULTISET | MAP => + s"$rowTerm.getMap($indexTerm)" + case ROW | STRUCTURED_TYPE => + s"$rowTerm.getRow($indexTerm, ${getFieldCount(t)})" + case DISTINCT_TYPE => + rowFieldReadAccess(ctx, indexTerm, rowTerm, t.asInstanceOf[DistinctType].getSourceType) + case RAW => + s"(($BINARY_RAW_VALUE) $rowTerm.getRawValue($indexTerm))" + case NULL | SYMBOL | UNRESOLVED => + throw new IllegalArgumentException("Illegal type: " + t) } // -------------------------- RowData Set Field ------------------------------- @@ -549,14 +563,22 @@ object CodeGenUtils { def binaryRowSetNull(index: Int, rowTerm: String, t: LogicalType): String = binaryRowSetNull(index.toString, rowTerm, t) - def binaryRowSetNull(indexTerm: String, rowTerm: String, t: LogicalType): String = t match { - case d: DecimalType if !DecimalData.isCompact(d.getPrecision) => - s"$rowTerm.setDecimal($indexTerm, null, ${d.getPrecision})" - case d: TimestampType if !TimestampData.isCompact(d.getPrecision) => - s"$rowTerm.setTimestamp($indexTerm, null, ${d.getPrecision})" - case d: LocalZonedTimestampType if !TimestampData.isCompact(d.getPrecision) => - s"$rowTerm.setTimestamp($indexTerm, null, ${d.getPrecision})" - case _ => s"$rowTerm.setNullAt($indexTerm)" + @tailrec + def binaryRowSetNull( + indexTerm: String, + rowTerm: String, + t: LogicalType) + : String = t.getTypeRoot match { + // ordered by type root definition + case DECIMAL if !DecimalData.isCompact(getPrecision(t)) => + s"$rowTerm.setDecimal($indexTerm, null, ${getPrecision(t)})" + case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE + if !TimestampData.isCompact(getPrecision(t)) => + s"$rowTerm.setTimestamp($indexTerm, null, ${getPrecision(t)})" + case DISTINCT_TYPE => + binaryRowSetNull(indexTerm, rowTerm, t.asInstanceOf[DistinctType].getSourceType) + case _ => + s"$rowTerm.setNullAt($indexTerm)" } def binaryRowFieldSetAccess( @@ -566,75 +588,102 @@ object CodeGenUtils { fieldValTerm: String): String = binaryRowFieldSetAccess(index.toString, binaryRowTerm, fieldType, fieldValTerm) + @tailrec def binaryRowFieldSetAccess( index: String, binaryRowTerm: String, t: LogicalType, - fieldValTerm: String): String = - t.getTypeRoot match { - case INTEGER => s"$binaryRowTerm.setInt($index, $fieldValTerm)" - case BIGINT => s"$binaryRowTerm.setLong($index, $fieldValTerm)" - case SMALLINT => s"$binaryRowTerm.setShort($index, $fieldValTerm)" - case TINYINT => s"$binaryRowTerm.setByte($index, $fieldValTerm)" - case FLOAT => s"$binaryRowTerm.setFloat($index, $fieldValTerm)" - case DOUBLE => s"$binaryRowTerm.setDouble($index, $fieldValTerm)" - case BOOLEAN => s"$binaryRowTerm.setBoolean($index, $fieldValTerm)" - case DATE => s"$binaryRowTerm.setInt($index, $fieldValTerm)" - case TIME_WITHOUT_TIME_ZONE => s"$binaryRowTerm.setInt($index, $fieldValTerm)" - case TIMESTAMP_WITHOUT_TIME_ZONE => - val dt = t.asInstanceOf[TimestampType] - s"$binaryRowTerm.setTimestamp($index, $fieldValTerm, ${dt.getPrecision})" - case TIMESTAMP_WITH_LOCAL_TIME_ZONE => - val dt = t.asInstanceOf[LocalZonedTimestampType] - s"$binaryRowTerm.setTimestamp($index, $fieldValTerm, ${dt.getPrecision})" - case INTERVAL_YEAR_MONTH => s"$binaryRowTerm.setInt($index, $fieldValTerm)" - case INTERVAL_DAY_TIME => s"$binaryRowTerm.setLong($index, $fieldValTerm)" - case DECIMAL => - val dt = t.asInstanceOf[DecimalType] - s"$binaryRowTerm.setDecimal($index, $fieldValTerm, ${dt.getPrecision})" - case _ => - throw new CodeGenException("Fail to find binary row field setter method of LogicalType " - + t + ".") - } + fieldValTerm: String) + : String = t.getTypeRoot match { + // ordered by type root definition + case BOOLEAN => + s"$binaryRowTerm.setBoolean($index, $fieldValTerm)" + case DECIMAL => + s"$binaryRowTerm.setDecimal($index, $fieldValTerm, ${getPrecision(t)})" + case TINYINT => + s"$binaryRowTerm.setByte($index, $fieldValTerm)" + case SMALLINT => + s"$binaryRowTerm.setShort($index, $fieldValTerm)" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => + s"$binaryRowTerm.setInt($index, $fieldValTerm)" + case BIGINT | INTERVAL_DAY_TIME => + s"$binaryRowTerm.setLong($index, $fieldValTerm)" + case FLOAT => + s"$binaryRowTerm.setFloat($index, $fieldValTerm)" + case DOUBLE => + s"$binaryRowTerm.setDouble($index, $fieldValTerm)" + case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => + s"$binaryRowTerm.setTimestamp($index, $fieldValTerm, ${getPrecision(t)})" + case DISTINCT_TYPE => + binaryRowFieldSetAccess( + index, + binaryRowTerm, + t.asInstanceOf[DistinctType].getSourceType, + fieldValTerm) + case _ => + throw new CodeGenException( + "Fail to find binary row field setter method of LogicalType " + t + ".") + } // -------------------------- BoxedWrapperRowData Set Field ------------------------------- + @tailrec def boxedWrapperRowFieldSetAccess( rowTerm: String, indexTerm: String, fieldTerm: String, - t: LogicalType): String = - t.getTypeRoot match { - case INTEGER => s"$rowTerm.setInt($indexTerm, $fieldTerm)" - case BIGINT => s"$rowTerm.setLong($indexTerm, $fieldTerm)" - case SMALLINT => s"$rowTerm.setShort($indexTerm, $fieldTerm)" - case TINYINT => s"$rowTerm.setByte($indexTerm, $fieldTerm)" - case FLOAT => s"$rowTerm.setFloat($indexTerm, $fieldTerm)" - case DOUBLE => s"$rowTerm.setDouble($indexTerm, $fieldTerm)" - case BOOLEAN => s"$rowTerm.setBoolean($indexTerm, $fieldTerm)" - case DATE => s"$rowTerm.setInt($indexTerm, $fieldTerm)" - case TIME_WITHOUT_TIME_ZONE => s"$rowTerm.setInt($indexTerm, $fieldTerm)" - case INTERVAL_YEAR_MONTH => s"$rowTerm.setInt($indexTerm, $fieldTerm)" - case INTERVAL_DAY_TIME => s"$rowTerm.setLong($indexTerm, $fieldTerm)" - case _ => s"$rowTerm.setNonPrimitiveValue($indexTerm, $fieldTerm)" - } + t: LogicalType) + : String = t.getTypeRoot match { + // ordered by type root definition + case BOOLEAN => + s"$rowTerm.setBoolean($indexTerm, $fieldTerm)" + case TINYINT => + s"$rowTerm.setByte($indexTerm, $fieldTerm)" + case SMALLINT => + s"$rowTerm.setShort($indexTerm, $fieldTerm)" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => + s"$rowTerm.setInt($indexTerm, $fieldTerm)" + case BIGINT | INTERVAL_DAY_TIME => + s"$rowTerm.setLong($indexTerm, $fieldTerm)" + case FLOAT => + s"$rowTerm.setFloat($indexTerm, $fieldTerm)" + case DOUBLE => + s"$rowTerm.setDouble($indexTerm, $fieldTerm)" + case DISTINCT_TYPE => + boxedWrapperRowFieldSetAccess( + rowTerm, + indexTerm, + fieldTerm, + t.asInstanceOf[DistinctType].getSourceType) + case _ => + s"$rowTerm.setNonPrimitiveValue($indexTerm, $fieldTerm)" + } // -------------------------- BinaryArray Set Access ------------------------------- + @tailrec def binaryArraySetNull( index: Int, arrayTerm: String, - t: LogicalType): String = t.getTypeRoot match { - case BOOLEAN => s"$arrayTerm.setNullBoolean($index)" - case TINYINT => s"$arrayTerm.setNullByte($index)" - case SMALLINT => s"$arrayTerm.setNullShort($index)" - case INTEGER => s"$arrayTerm.setNullInt($index)" - case FLOAT => s"$arrayTerm.setNullFloat($index)" - case DOUBLE => s"$arrayTerm.setNullDouble($index)" - case TIME_WITHOUT_TIME_ZONE => s"$arrayTerm.setNullInt($index)" - case DATE => s"$arrayTerm.setNullInt($index)" - case INTERVAL_YEAR_MONTH => s"$arrayTerm.setNullInt($index)" - case _ => s"$arrayTerm.setNullLong($index)" + t: LogicalType) + : String = t.getTypeRoot match { + // ordered by type root definition + case BOOLEAN => + s"$arrayTerm.setNullBoolean($index)" + case TINYINT => + s"$arrayTerm.setNullByte($index)" + case SMALLINT => + s"$arrayTerm.setNullShort($index)" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => + s"$arrayTerm.setNullInt($index)" + case FLOAT => + s"$arrayTerm.setNullFloat($index)" + case DOUBLE => + s"$arrayTerm.setNullDouble($index)" + case DISTINCT_TYPE => + binaryArraySetNull(index, arrayTerm, t) + case _ => + s"$arrayTerm.setNullLong($index)" } // -------------------------- BinaryWriter Write ------------------------------- @@ -642,17 +691,22 @@ object CodeGenUtils { def binaryWriterWriteNull(index: Int, writerTerm: String, t: LogicalType): String = binaryWriterWriteNull(index.toString, writerTerm, t) + @tailrec def binaryWriterWriteNull( indexTerm: String, writerTerm: String, - t: LogicalType): String = t match { - case d: DecimalType if !DecimalData.isCompact(d.getPrecision) => - s"$writerTerm.writeDecimal($indexTerm, null, ${d.getPrecision})" - case d: TimestampType if !TimestampData.isCompact(d.getPrecision) => - s"$writerTerm.writeTimestamp($indexTerm, null, ${d.getPrecision})" - case d: LocalZonedTimestampType if !TimestampData.isCompact(d.getPrecision) => - s"$writerTerm.writeTimestamp($indexTerm, null, ${d.getPrecision})" - case _ => s"$writerTerm.setNullAt($indexTerm)" + t: LogicalType) + : String = t.getTypeRoot match { + // ordered by type root definition + case DECIMAL if !DecimalData.isCompact(getPrecision(t)) => + s"$writerTerm.writeDecimal($indexTerm, null, ${getPrecision(t)})" + case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE + if !TimestampData.isCompact(getPrecision(t)) => + s"$writerTerm.writeTimestamp($indexTerm, null, ${getPrecision(t)})" + case DISTINCT_TYPE => + binaryWriterWriteNull(indexTerm, writerTerm, t.asInstanceOf[DistinctType].getSourceType) + case _ => + s"$writerTerm.setNullAt($indexTerm)" } def binaryWriterWriteField( @@ -663,50 +717,74 @@ object CodeGenUtils { fieldType: LogicalType): String = binaryWriterWriteField(ctx, index.toString, fieldValTerm, writerTerm, fieldType) + @tailrec def binaryWriterWriteField( ctx: CodeGeneratorContext, indexTerm: String, fieldValTerm: String, writerTerm: String, - t: LogicalType): String = - t.getTypeRoot match { - case INTEGER => s"$writerTerm.writeInt($indexTerm, $fieldValTerm)" - case BIGINT => s"$writerTerm.writeLong($indexTerm, $fieldValTerm)" - case SMALLINT => s"$writerTerm.writeShort($indexTerm, $fieldValTerm)" - case TINYINT => s"$writerTerm.writeByte($indexTerm, $fieldValTerm)" - case FLOAT => s"$writerTerm.writeFloat($indexTerm, $fieldValTerm)" - case DOUBLE => s"$writerTerm.writeDouble($indexTerm, $fieldValTerm)" - case BOOLEAN => s"$writerTerm.writeBoolean($indexTerm, $fieldValTerm)" - case VARBINARY | BINARY => s"$writerTerm.writeBinary($indexTerm, $fieldValTerm)" - case VARCHAR | CHAR => s"$writerTerm.writeString($indexTerm, $fieldValTerm)" - case DECIMAL => - val dt = t.asInstanceOf[DecimalType] - s"$writerTerm.writeDecimal($indexTerm, $fieldValTerm, ${dt.getPrecision})" - case DATE => s"$writerTerm.writeInt($indexTerm, $fieldValTerm)" - case TIME_WITHOUT_TIME_ZONE => s"$writerTerm.writeInt($indexTerm, $fieldValTerm)" - case TIMESTAMP_WITHOUT_TIME_ZONE => - val dt = t.asInstanceOf[TimestampType] - s"$writerTerm.writeTimestamp($indexTerm, $fieldValTerm, ${dt.getPrecision})" - case TIMESTAMP_WITH_LOCAL_TIME_ZONE => - val dt = t.asInstanceOf[LocalZonedTimestampType] - s"$writerTerm.writeTimestamp($indexTerm, $fieldValTerm, ${dt.getPrecision})" - case INTERVAL_YEAR_MONTH => s"$writerTerm.writeInt($indexTerm, $fieldValTerm)" - case INTERVAL_DAY_TIME => s"$writerTerm.writeLong($indexTerm, $fieldValTerm)" - - // complex types - case ARRAY => - val ser = ctx.addReusableTypeSerializer(t) - s"$writerTerm.writeArray($indexTerm, $fieldValTerm, $ser)" - case MULTISET | MAP => - val ser = ctx.addReusableTypeSerializer(t) - s"$writerTerm.writeMap($indexTerm, $fieldValTerm, $ser)" - case ROW => - val ser = ctx.addReusableTypeSerializer(t) - s"$writerTerm.writeRow($indexTerm, $fieldValTerm, $ser)" - case RAW => - val ser = ctx.addReusableTypeSerializer(t) - s"$writerTerm.writeRawValue($indexTerm, $fieldValTerm, $ser)" - } + t: LogicalType) + : String = t.getTypeRoot match { + // ordered by type root definition + case CHAR | VARCHAR => + s"$writerTerm.writeString($indexTerm, $fieldValTerm)" + case BOOLEAN => + s"$writerTerm.writeBoolean($indexTerm, $fieldValTerm)" + case BINARY | VARBINARY => + s"$writerTerm.writeBinary($indexTerm, $fieldValTerm)" + case DECIMAL => + s"$writerTerm.writeDecimal($indexTerm, $fieldValTerm, ${getPrecision(t)})" + case TINYINT => + s"$writerTerm.writeByte($indexTerm, $fieldValTerm)" + case SMALLINT => + s"$writerTerm.writeShort($indexTerm, $fieldValTerm)" + case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => + s"$writerTerm.writeInt($indexTerm, $fieldValTerm)" + case BIGINT | INTERVAL_DAY_TIME => + s"$writerTerm.writeLong($indexTerm, $fieldValTerm)" + case FLOAT => + s"$writerTerm.writeFloat($indexTerm, $fieldValTerm)" + case DOUBLE => + s"$writerTerm.writeDouble($indexTerm, $fieldValTerm)" + case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => + s"$writerTerm.writeTimestamp($indexTerm, $fieldValTerm, ${getPrecision(t)})" + case TIMESTAMP_WITH_TIME_ZONE => + throw new UnsupportedOperationException("Unsupported type: " + t) + case ARRAY => + val ser = ctx.addReusableTypeSerializer(t) + s"$writerTerm.writeArray($indexTerm, $fieldValTerm, $ser)" + case MULTISET | MAP => + val ser = ctx.addReusableTypeSerializer(t) + s"$writerTerm.writeMap($indexTerm, $fieldValTerm, $ser)" + case ROW | STRUCTURED_TYPE => + val ser = ctx.addReusableTypeSerializer(t) + s"$writerTerm.writeRow($indexTerm, $fieldValTerm, $ser)" + case DISTINCT_TYPE => + binaryWriterWriteField( + ctx, + indexTerm, + fieldValTerm, + writerTerm, + t.asInstanceOf[DistinctType].getSourceType) + case RAW => + val ser = ctx.addReusableTypeSerializer(t) + s"$writerTerm.writeRawValue($indexTerm, $fieldValTerm, $ser)" + case NULL | SYMBOL | UNRESOLVED => + throw new IllegalArgumentException("Illegal type: " + t); + } + + // -------------------------- Data Structure Conversion ------------------------------- + + /** + * If it's internally compatible, don't need to DataStructure converter. + * clazz != classOf[Row] => Row can only infer GenericType[Row]. + */ + def isInternalClass(t: DataType): Boolean = { + val clazz = t.getConversionClass + clazz != classOf[Object] && clazz != classOf[Row] && + (classOf[RowData].isAssignableFrom(clazz) || + clazz == toInternalConversionClass(fromDataTypeToLogicalType(t))) + } private def isConverterIdentity(t: DataType): Boolean = { DataFormatConverters.getConverterForDataType(t).isInstanceOf[IdentityConverter[_]] @@ -808,9 +886,4 @@ object CodeGenUtils { s"${internalExpr.nullTerm} ? null : ($externalResultTerm)" } } - - def udfFieldName(udf: UserDefinedFunction): String = s"function_${udf.functionIdentifier}" - - def genLogInfo(logTerm: String, format: String, argTerm: String): String = - s"""$logTerm.info("$format", $argTerm);""" } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala index 174158d4b6f3c..850d55f573f3a 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/EqualiserCodeGenerator.scala @@ -24,8 +24,10 @@ import org.apache.flink.table.planner.codegen.calls.ScalarOperatorGens.generateE import org.apache.flink.table.runtime.generated.{GeneratedRecordEqualiser, RecordEqualiser} import org.apache.flink.table.runtime.types.PlannerTypeUtils import org.apache.flink.table.types.logical.LogicalTypeRoot._ -import org.apache.flink.table.types.logical.{LogicalType, RowType} +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{getFieldTypes, isCompositeType} +import org.apache.flink.table.types.logical.{DistinctType, LogicalType} +import scala.annotation.tailrec import scala.collection.JavaConverters._ class EqualiserCodeGenerator(fieldTypes: Array[LogicalType]) { @@ -57,9 +59,9 @@ class EqualiserCodeGenerator(fieldTypes: Array[LogicalType]) { // TODO merge ScalarOperatorGens.generateEquals. val (equalsCode, equalsResult) = if (isInternalPrimitive(fieldType)) { ("", s"$leftFieldTerm == $rightFieldTerm") - } else if (isRowData(fieldType)) { + } else if (isCompositeType(fieldType)) { val equaliserGenerator = new EqualiserCodeGenerator( - fieldType.asInstanceOf[RowType].getChildren.asScala.toArray) + getFieldTypes(fieldType).asScala.toArray) val generatedEqualiser = equaliserGenerator .generateRecordEqualiser("field$" + i + "GeneratedEqualiser") val generatedEqualiserTerm = ctx.addReusableObject( @@ -128,15 +130,14 @@ class EqualiserCodeGenerator(fieldTypes: Array[LogicalType]) { new GeneratedRecordEqualiser(className, functionCode, ctx.references.toArray) } + @tailrec private def isInternalPrimitive(t: LogicalType): Boolean = t.getTypeRoot match { case _ if PlannerTypeUtils.isPrimitive(t) => true - case DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH |INTERVAL_DAY_TIME => true - case _ => false - } + case DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH | INTERVAL_DAY_TIME => true + + case DISTINCT_TYPE => isInternalPrimitive(t.asInstanceOf[DistinctType].getSourceType) - private def isRowData(t: LogicalType): Boolean = t match { - case _: RowType => true case _ => false } } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala index 950a35b1f74ab..82a01224c4cee 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala @@ -30,11 +30,9 @@ import org.apache.flink.table.planner.codegen.FunctionCodeGenerator.generateFunc import org.apache.flink.table.planner.plan.utils.PythonUtil.containsPythonCall import org.apache.flink.table.types.logical.RowType import org.apache.flink.table.util.TimestampStringUtils.fromLocalDateTime - import org.apache.calcite.avatica.util.ByteString import org.apache.calcite.rex.{RexBuilder, RexExecutor, RexNode} import org.apache.calcite.sql.`type`.SqlTypeName - import java.io.File import scala.collection.JavaConverters._ @@ -72,7 +70,9 @@ class ExpressionReducer( // we don't support object literals yet, we skip those constant expressions case (SqlTypeName.ANY, _) | + (SqlTypeName.OTHER, _) | (SqlTypeName.ROW, _) | + (SqlTypeName.STRUCTURED, _) | (SqlTypeName.ARRAY, _) | (SqlTypeName.MAP, _) | (SqlTypeName.MULTISET, _) => None @@ -133,7 +133,9 @@ class ExpressionReducer( unreduced.getType.getSqlTypeName match { // we insert the original expression for object literals case SqlTypeName.ANY | + SqlTypeName.OTHER | SqlTypeName.ROW | + SqlTypeName.STRUCTURED | SqlTypeName.ARRAY | SqlTypeName.MAP | SqlTypeName.MULTISET => diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/GenerateUtils.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/GenerateUtils.scala index 4fc52d27baaca..9d0fe44e351cf 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/GenerateUtils.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/GenerateUtils.scala @@ -34,12 +34,13 @@ import org.apache.flink.table.planner.codegen.CodeGenUtils._ import org.apache.flink.table.planner.codegen.GeneratedExpression.{ALWAYS_NULL, NEVER_NULL, NO_CODE} import org.apache.flink.table.planner.codegen.calls.CurrentTimePointCallGen import org.apache.flink.table.planner.plan.utils.SortUtil -import org.apache.flink.table.runtime.types.PlannerTypeUtils import org.apache.flink.table.runtime.typeutils.TypeCheckUtils.{isCharacterString, isReference, isTemporal} import org.apache.flink.table.types.logical.LogicalTypeRoot._ import org.apache.flink.table.types.logical._ +import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{getFieldCount, getFieldTypes} import org.apache.flink.table.util.TimestampStringUtils.toLocalDateTime +import scala.annotation.tailrec import scala.collection.mutable /** @@ -209,39 +210,47 @@ object GenerateUtils { /** * Generates a record declaration statement. The record can be any type of RowData or * other types. + * * @param t the record type * @param clazz the specified class of the type (only used when RowType) * @param recordTerm the record term to be declared * @param recordWriterTerm the record writer term (only used when BinaryRowData type) * @return the record declaration statement - */ + */ + @tailrec def generateRecordStatement( t: LogicalType, clazz: Class[_], recordTerm: String, - recordWriterTerm: Option[String] = None): String = { - t match { - case rt: RowType if clazz == classOf[BinaryRowData] => - val writerTerm = recordWriterTerm.getOrElse( - throw new CodeGenException("No writer is specified when writing BinaryRowData record.") - ) - val binaryRowWriter = className[BinaryRowWriter] - val typeTerm = clazz.getCanonicalName - s""" - |final $typeTerm $recordTerm = new $typeTerm(${rt.getFieldCount}); - |final $binaryRowWriter $writerTerm = new $binaryRowWriter($recordTerm); - |""".stripMargin.trim - case rt: RowType if clazz == classOf[GenericRowData] || - clazz == classOf[BoxedWrapperRowData] => - val typeTerm = clazz.getCanonicalName - s"final $typeTerm $recordTerm = new $typeTerm(${rt.getFieldCount});" - case _: RowType if clazz == classOf[JoinedRowData] => - val typeTerm = clazz.getCanonicalName - s"final $typeTerm $recordTerm = new $typeTerm();" - case _ => - val typeTerm = boxedTypeTermForType(t) - s"final $typeTerm $recordTerm = new $typeTerm();" - } + recordWriterTerm: Option[String] = None) + : String = t.getTypeRoot match { + // ordered by type root definition + case ROW | STRUCTURED_TYPE if clazz == classOf[BinaryRowData] => + val writerTerm = recordWriterTerm.getOrElse( + throw new CodeGenException("No writer is specified when writing BinaryRowData record.") + ) + val binaryRowWriter = className[BinaryRowWriter] + val typeTerm = clazz.getCanonicalName + s""" + |final $typeTerm $recordTerm = new $typeTerm(${getFieldCount(t)}); + |final $binaryRowWriter $writerTerm = new $binaryRowWriter($recordTerm); + |""".stripMargin.trim + case ROW | STRUCTURED_TYPE if clazz == classOf[GenericRowData] || + clazz == classOf[BoxedWrapperRowData] => + val typeTerm = clazz.getCanonicalName + s"final $typeTerm $recordTerm = new $typeTerm(${getFieldCount(t)});" + case ROW | STRUCTURED_TYPE if clazz == classOf[JoinedRowData] => + val typeTerm = clazz.getCanonicalName + s"final $typeTerm $recordTerm = new $typeTerm();" + case DISTINCT_TYPE => + generateRecordStatement( + t.asInstanceOf[DistinctType].getSourceType, + clazz, + recordTerm, + recordWriterTerm) + case _ => + val typeTerm = boxedTypeTermForType(t) + s"final $typeTerm $recordTerm = new $typeTerm();" } def generateNullLiteral( @@ -273,6 +282,7 @@ object GenerateUtils { literalValue = Some(literalValue)) } + @tailrec def generateLiteral( ctx: CodeGeneratorContext, literalType: LogicalType, @@ -282,10 +292,41 @@ object GenerateUtils { } // non-null values literalType.getTypeRoot match { + // ordered by type root definition + case CHAR | VARCHAR => + val escapedValue = StringEscapeUtils.ESCAPE_JAVA.translate(literalValue.toString) + val field = ctx.addReusableStringConstants(escapedValue) + generateNonNullLiteral(literalType, field, StringData.fromString(escapedValue)) case BOOLEAN => generateNonNullLiteral(literalType, literalValue.toString, literalValue) + case BINARY | VARBINARY => + val bytesVal = literalValue.asInstanceOf[ByteString].getBytes + val fieldTerm = ctx.addReusableObject( + bytesVal, "binary", bytesVal.getClass.getCanonicalName) + generateNonNullLiteral(literalType, fieldTerm, bytesVal) + + case DECIMAL => + val dt = literalType.asInstanceOf[DecimalType] + val precision = dt.getPrecision + val scale = dt.getScale + val fieldTerm = newName("decimal") + val decimalClass = className[DecimalData] + val fieldDecimal = + s""" + |$decimalClass $fieldTerm = + | $DECIMAL_UTIL.castFrom("${literalValue.toString}", $precision, $scale); + |""".stripMargin + ctx.addReusableMember(fieldDecimal) + val value = DecimalData.fromBigDecimal( + literalValue.asInstanceOf[JBigDecimal], precision, scale) + if (value == null) { + generateNullLiteral(literalType, ctx.nullCheck) + } else { + generateNonNullLiteral(literalType, fieldTerm, value) + } + case TINYINT => val decimal = BigDecimal(literalValue.asInstanceOf[JBigDecimal]) generateNonNullLiteral(literalType, decimal.byteValue().toString, decimal.byteValue()) @@ -335,36 +376,6 @@ object GenerateUtils { case _ => generateNonNullLiteral( literalType, doubleValue.toString + "d", doubleValue) } - case DECIMAL => - val dt = literalType.asInstanceOf[DecimalType] - val precision = dt.getPrecision - val scale = dt.getScale - val fieldTerm = newName("decimal") - val decimalClass = className[DecimalData] - val fieldDecimal = - s""" - |$decimalClass $fieldTerm = - | $DECIMAL_UTIL.castFrom("${literalValue.toString}", $precision, $scale); - |""".stripMargin - ctx.addReusableMember(fieldDecimal) - val value = DecimalData.fromBigDecimal( - literalValue.asInstanceOf[JBigDecimal], precision, scale) - if (value == null) { - generateNullLiteral(literalType, ctx.nullCheck) - } else { - generateNonNullLiteral(literalType, fieldTerm, value) - } - - case VARCHAR | CHAR => - val escapedValue = StringEscapeUtils.ESCAPE_JAVA.translate(literalValue.toString) - val field = ctx.addReusableStringConstants(escapedValue) - generateNonNullLiteral(literalType, field, StringData.fromString(escapedValue)) - - case VARBINARY | BINARY => - val bytesVal = literalValue.asInstanceOf[ByteString].getBytes - val fieldTerm = ctx.addReusableObject( - bytesVal, "binary", bytesVal.getClass.getCanonicalName) - generateNonNullLiteral(literalType, fieldTerm, bytesVal) case DATE => generateNonNullLiteral(literalType, literalValue.toString, literalValue) @@ -384,6 +395,9 @@ object GenerateUtils { ctx.addReusableMember(fieldTimestamp) generateNonNullLiteral(literalType, fieldTerm, ts) + case TIMESTAMP_WITH_TIME_ZONE => + throw new UnsupportedOperationException("Unsupported type: " + literalType) + case TIMESTAMP_WITH_LOCAL_TIME_ZONE => val fieldTerm = newName("timestampWithLocalZone") val ins = @@ -420,13 +434,19 @@ object GenerateUtils { s"Decimal '$decimal' can not be converted to interval of milliseconds.") } + case DISTINCT_TYPE => + generateLiteral(ctx, literalType.asInstanceOf[DistinctType].getSourceType, literalValue) + // Symbol type for special flags e.g. TRIM's BOTH, LEADING, TRAILING case RAW if literalType.asInstanceOf[TypeInformationRawType[_]] .getTypeInformation.getTypeClass.isAssignableFrom(classOf[Enum[_]]) => generateSymbol(literalValue.asInstanceOf[Enum[_]]) - case t@_ => - throw new CodeGenException(s"Type not supported: $t") + case SYMBOL => + throw new UnsupportedOperationException() // TODO support symbol? + + case ARRAY | MULTISET | MAP | ROW | STRUCTURED_TYPE | NULL | UNRESOLVED => + throw new CodeGenException(s"Type not supported: $literalType") } } @@ -546,10 +566,15 @@ object GenerateUtils { index: Int, deepCopy: Boolean = false): GeneratedExpression = { - val fieldType = inputType match { - case ct: RowType => ct.getTypeAt(index) - case _ => inputType + @tailrec + def getFieldType(t: LogicalType, pos: Int): LogicalType = t.getTypeRoot match { + // ordered by type root definition + case ROW | STRUCTURED_TYPE => t.getChildren.get(pos) + case DISTINCT_TYPE => getFieldType(t.asInstanceOf[DistinctType].getSourceType, pos) + case _ => t } + + val fieldType = getFieldType(inputType, index) val resultTypeTerm = primitiveTypeTermForType(fieldType) val defaultValue = primitiveDefaultValue(fieldType) val Seq(resultTerm, nullTerm) = ctx.addReusableLocalVariables( @@ -636,14 +661,16 @@ object GenerateUtils { } } + @tailrec def generateFieldAccess( ctx: CodeGeneratorContext, inputType: LogicalType, inputTerm: String, - index: Int): GeneratedExpression = - inputType match { - case ct: RowType => - val fieldType = ct.getTypeAt(index) + index: Int) + : GeneratedExpression = inputType.getTypeRoot match { + // ordered by type root definition + case ROW | STRUCTURED_TYPE => + val fieldType = getFieldTypes(inputType).get(index) val resultTypeTerm = primitiveTypeTermForType(fieldType) val defaultValue = primitiveDefaultValue(fieldType) val readCode = rowFieldReadAccess(ctx, index.toString, inputTerm, fieldType) @@ -667,6 +694,13 @@ object GenerateUtils { } GeneratedExpression(fieldTerm, nullTerm, inputCode, fieldType) + case DISTINCT_TYPE => + generateFieldAccess( + ctx, + inputType.asInstanceOf[DistinctType].getSourceType, + inputTerm, + index) + case _ => val fieldTypeTerm = boxedTypeTermForType(inputType) val inputCode = s"($fieldTypeTerm) $inputTerm" @@ -674,23 +708,30 @@ object GenerateUtils { } /** - * Generates code for comparing two field. + * Generates code for comparing two fields. */ + @tailrec def generateCompare( ctx: CodeGeneratorContext, t: LogicalType, nullsIsLast: Boolean, leftTerm: String, - rightTerm: String): String = t.getTypeRoot match { - case BOOLEAN => s"($leftTerm == $rightTerm ? 0 : ($leftTerm ? 1 : -1))" - case DATE | TIME_WITHOUT_TIME_ZONE => - s"($leftTerm > $rightTerm ? 1 : $leftTerm < $rightTerm ? -1 : 0)" - case _ if PlannerTypeUtils.isPrimitive(t) => - s"($leftTerm > $rightTerm ? 1 : $leftTerm < $rightTerm ? -1 : 0)" - case VARBINARY | BINARY => + rightTerm: String) + : String = t.getTypeRoot match { + // ordered by type root definition + case CHAR | VARCHAR | DECIMAL | TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => + s"$leftTerm.compareTo($rightTerm)" + case BOOLEAN => + s"($leftTerm == $rightTerm ? 0 : ($leftTerm ? 1 : -1))" + case BINARY | VARBINARY => val sortUtil = classOf[org.apache.flink.table.runtime.operators.sort.SortUtil] .getCanonicalName s"$sortUtil.compareBinary($leftTerm, $rightTerm)" + case TINYINT | SMALLINT | INTEGER | BIGINT | FLOAT | DOUBLE | DATE | TIME_WITHOUT_TIME_ZONE | + INTERVAL_YEAR_MONTH | INTERVAL_DAY_TIME => + s"($leftTerm > $rightTerm ? 1 : $leftTerm < $rightTerm ? -1 : 0)" + case TIMESTAMP_WITH_TIME_ZONE | MULTISET | MAP => + throw new UnsupportedOperationException() // TODO support MULTISET and MAP? case ARRAY => val at = t.asInstanceOf[ArrayType] val compareFunc = newName("compareArray") @@ -706,13 +747,13 @@ object GenerateUtils { """ ctx.addReusableMember(funcCode) s"$compareFunc($leftTerm, $rightTerm)" - case ROW => - val rowType = t.asInstanceOf[RowType] - val orders = (0 until rowType.getFieldCount).map(_ => true).toArray + case ROW | STRUCTURED_TYPE => + val fieldCount = getFieldCount(t) + val orders = (0 until fieldCount).map(_ => true).toArray val comparisons = generateRowCompare( ctx, - (0 until rowType.getFieldCount).toArray, - rowType.getChildren.toArray(Array[LogicalType]()), + (0 until fieldCount).toArray, + getFieldTypes(t).toArray(Array[LogicalType]()), orders, SortUtil.getNullDefaultOrders(orders), "a", @@ -727,18 +768,38 @@ object GenerateUtils { """ ctx.addReusableMember(funcCode) s"$compareFunc($leftTerm, $rightTerm)" + case DISTINCT_TYPE => + generateCompare( + ctx, + t.asInstanceOf[DistinctType].getSourceType, + nullsIsLast, + leftTerm, + rightTerm) case RAW => - val rawType = t.asInstanceOf[TypeInformationRawType[_]] - val ser = ctx.addReusableObject( - rawType.getTypeInformation.createSerializer(new ExecutionConfig), "serializer") - val comp = ctx.addReusableObject( - rawType.getTypeInformation.asInstanceOf[AtomicTypeInfo[_]] - .createComparator(true, new ExecutionConfig), - "comparator") - s""" - |$comp.compare($leftTerm.toObject($ser), $rightTerm.toObject($ser)) - """.stripMargin - case other => s"$leftTerm.compareTo($rightTerm)" + t match { + case rawType: RawType[_] => + val clazz = rawType.getOriginatingClass + if (!classOf[Comparable[_]].isAssignableFrom(clazz)) { + throw new CodeGenException( + s"Raw type class '$clazz' must implement ${className[Comparable[_]]} to be used " + + s"in a comparision of two '${rawType.asSummaryString()}' types.") + } + val serializer = rawType.getTypeSerializer + val serializerTerm = ctx.addReusableObject(serializer, "serializer") + s"((${className[Comparable[_]]}) $leftTerm.toObject($serializerTerm))" + + s".compareTo($rightTerm.toObject($serializerTerm))" + + case rawType: TypeInformationRawType[_] => + val serializer = rawType.getTypeInformation.createSerializer(new ExecutionConfig) + val ser = ctx.addReusableObject(serializer, "serializer") + val comp = ctx.addReusableObject( + rawType.getTypeInformation.asInstanceOf[AtomicTypeInfo[_]] + .createComparator(true, new ExecutionConfig), + "comparator") + s"$comp.compare($leftTerm.toObject($ser), $rightTerm.toObject($ser))" + } + case NULL | SYMBOL | UNRESOLVED => + throw new IllegalArgumentException("Illegal type: " + t) } /** diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/AggCodeGenHelper.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/AggCodeGenHelper.scala index 7493aa2450b3f..1fbacc07878b7 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/AggCodeGenHelper.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/agg/batch/AggCodeGenHelper.scala @@ -18,7 +18,6 @@ package org.apache.flink.table.planner.codegen.agg.batch -import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.runtime.util.SingleElementIterator import org.apache.flink.streaming.api.operators.OneInputStreamOperator import org.apache.flink.table.data.{GenericRowData, RowData} @@ -39,12 +38,13 @@ import org.apache.flink.table.runtime.types.InternalSerializers import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.{fromDataTypeToLogicalType, fromLogicalTypeToDataType} import org.apache.flink.table.types.DataType import org.apache.flink.table.types.logical.LogicalTypeRoot._ -import org.apache.flink.table.types.logical.{LogicalType, RowType} - +import org.apache.flink.table.types.logical.{DistinctType, LogicalType, RowType} import org.apache.calcite.rel.core.AggregateCall import org.apache.calcite.rex.RexNode import org.apache.calcite.tools.RelBuilder +import scala.annotation.tailrec + /** * Batch aggregate code generate helper. */ @@ -360,16 +360,7 @@ object AggCodeGenHelper { aggBufferExprs.zip(initAggBufferExprs).map { case (aggBufVar, initExpr) => - val resultCode = aggBufVar.resultType.getTypeRoot match { - case VARCHAR | CHAR | ROW | ARRAY | MULTISET | MAP => - val serializer = InternalSerializers.create( - aggBufVar.resultType, new ExecutionConfig) - val term = ctx.addReusableObject( - serializer, "serializer", serializer.getClass.getCanonicalName) - val typeTerm = boxedTypeTermForType(aggBufVar.resultType) - s"($typeTerm) $term.copy(${initExpr.resultTerm})" - case _ => initExpr.resultTerm - } + val resultCode = genElementCopyTerm(ctx, aggBufVar.resultType, initExpr.resultTerm) s""" |${initExpr.code} |${aggBufVar.nullTerm} = ${initExpr.nullTerm}; @@ -378,6 +369,23 @@ object AggCodeGenHelper { } mkString "\n" } + @tailrec + private def genElementCopyTerm( + ctx: CodeGeneratorContext, + t: LogicalType, + inputTerm: String) + : String = t.getTypeRoot match { + case CHAR | VARCHAR | ARRAY | MULTISET | MAP | ROW | STRUCTURED_TYPE => + val serializer = InternalSerializers.create(t) + val term = ctx.addReusableObject( + serializer, "serializer", serializer.getClass.getCanonicalName) + val typeTerm = boxedTypeTermForType(t) + s"($typeTerm) $term.copy($inputTerm)" + case DISTINCT_TYPE => + genElementCopyTerm(ctx, t.asInstanceOf[DistinctType].getSourceType, inputTerm) + case _ => inputTerm + } + private[flink] def genAggregateByFlatAggregateBuffer( isMerge: Boolean, ctx: CodeGeneratorContext, diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/typeutils/TypeCheckUtils.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/typeutils/TypeCheckUtils.java index 319021d940252..9a588820047a9 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/typeutils/TypeCheckUtils.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/typeutils/TypeCheckUtils.java @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime.typeutils; +import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeFamily; import org.apache.flink.table.types.logical.TimestampKind; @@ -60,9 +61,10 @@ public static boolean isProcTime(LogicalType type) { } public static boolean isTimeInterval(LogicalType type) { + // ordered by type root definition switch (type.getTypeRoot()) { - case INTERVAL_DAY_TIME: case INTERVAL_YEAR_MONTH: + case INTERVAL_DAY_TIME: return true; default: return false; @@ -122,22 +124,28 @@ public static boolean isComparable(LogicalType type) { } public static boolean isMutable(LogicalType type) { - // the internal representation of String is StringData which is mutable + // ordered by type root definition switch (type.getTypeRoot()) { - case VARCHAR: case CHAR: + case VARCHAR: // the internal representation of String is StringData which is mutable case ARRAY: case MULTISET: case MAP: case ROW: + case STRUCTURED_TYPE: case RAW: return true; + case TIMESTAMP_WITH_TIME_ZONE: + throw new UnsupportedOperationException("Unsupported type: " + type); + case DISTINCT_TYPE: + return isMutable(((DistinctType) type).getSourceType()); default: return false; } } public static boolean isReference(LogicalType type) { + // ordered by type root definition switch (type.getTypeRoot()) { case BOOLEAN: case TINYINT: @@ -153,6 +161,10 @@ public static boolean isReference(LogicalType type) { case INTERVAL_YEAR_MONTH: case INTERVAL_DAY_TIME: return false; + case TIMESTAMP_WITH_TIME_ZONE: + throw new UnsupportedOperationException("Unsupported type: " + type); + case DISTINCT_TYPE: + return isReference(((DistinctType) type).getSourceType()); default: return true; } From f8b8150b48d5c35f1b294b5d8f27ee5bba8fead5 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 18 May 2020 13:03:59 +0200 Subject: [PATCH 070/773] [hotfix][table] Update FLIP-65 functions to new data structure converters --- .../table/types/utils/DataTypeUtils.java | 10 ++ .../table/types/utils/DataTypeUtilsTest.java | 12 ++ .../table/planner/codegen/CodeGenUtils.scala | 134 +++++++++++++++++- .../codegen/CodeGeneratorContext.scala | 33 ++++- .../planner/codegen/ExpressionReducer.scala | 9 ++ .../calls/BridgingSqlFunctionCallGen.scala | 30 ++-- .../ArrayBooleanArrayConverter.java | 2 +- .../conversion/ArrayByteArrayConverter.java | 2 +- .../conversion/ArrayDoubleArrayConverter.java | 2 +- .../conversion/ArrayFloatArrayConverter.java | 2 +- .../conversion/ArrayIntArrayConverter.java | 2 +- .../conversion/ArrayLongArrayConverter.java | 2 +- .../conversion/ArrayObjectArrayConverter.java | 2 +- .../conversion/ArrayShortArrayConverter.java | 2 +- .../data/conversion/DateDateConverter.java | 2 +- .../conversion/DateLocalDateConverter.java | 2 +- .../DayTimeIntervalDurationConverter.java | 2 +- .../DecimalBigDecimalConverter.java | 2 +- .../data/conversion/IdentityConverter.java | 2 +- .../LocalZonedTimestampInstantConverter.java | 2 +- .../LocalZonedTimestampIntConverter.java | 2 +- .../LocalZonedTimestampLongConverter.java | 2 +- .../data/conversion/MapMapConverter.java | 2 +- .../conversion/RawByteArrayConverter.java | 2 +- .../data/conversion/RawObjectConverter.java | 2 +- .../data/conversion/RowRowConverter.java | 2 +- .../conversion/StringByteArrayConverter.java | 2 +- .../conversion/StringStringConverter.java | 2 +- .../conversion/StructuredObjectConverter.java | 2 +- .../conversion/TimeLocalTimeConverter.java | 2 +- .../data/conversion/TimeLongConverter.java | 2 +- .../data/conversion/TimeTimeConverter.java | 2 +- .../TimestampLocalDateTimeConverter.java | 2 +- .../TimestampTimestampConverter.java | 2 +- .../YearMonthIntervalPeriodConverter.java | 2 +- 35 files changed, 233 insertions(+), 53 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/DataTypeUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/DataTypeUtils.java index 84c824263fbed..b0c74d189cf0b 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/DataTypeUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/DataTypeUtils.java @@ -47,7 +47,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.apache.flink.table.types.extraction.ExtractionUtils.primitiveToWrapper; import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFieldNames; +import static org.apache.flink.table.types.logical.utils.LogicalTypeUtils.toInternalConversionClass; /** * Utilities for handling {@link DataType}s. @@ -55,6 +57,14 @@ @Internal public final class DataTypeUtils { + /** + * Checks whether a given data type is an internal data structure. + */ + public static boolean isInternal(DataType dataType) { + final Class clazz = primitiveToWrapper(dataType.getConversionClass()); + return clazz == toInternalConversionClass(dataType.getLogicalType()); + } + /** * Replaces the {@link LogicalType} of a {@link DataType}, i.e., it keeps the bridging class. */ diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/utils/DataTypeUtilsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/utils/DataTypeUtilsTest.java index c650a6097f236..07edff7d19892 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/utils/DataTypeUtilsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/utils/DataTypeUtilsTest.java @@ -23,6 +23,7 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.FieldsDataType; import org.apache.flink.table.types.logical.DistinctType; @@ -42,12 +43,23 @@ import static org.apache.flink.table.api.DataTypes.STRING; import static org.apache.flink.table.api.DataTypes.TIMESTAMP; import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Tests for {@link DataTypeUtils}. */ public class DataTypeUtilsTest { + + @Test + public void testIsInternalClass() { + assertTrue(DataTypeUtils.isInternal(DataTypes.INT())); + assertTrue(DataTypeUtils.isInternal(DataTypes.INT().notNull().bridgedTo(int.class))); + assertTrue(DataTypeUtils.isInternal(DataTypes.ROW().bridgedTo(RowData.class))); + assertFalse(DataTypeUtils.isInternal(DataTypes.ROW())); + } + @Test public void testExpandRowType() { DataType dataType = ROW( diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala index 6e62a3f4136d5..786d573adc6bc 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala @@ -27,6 +27,7 @@ import org.apache.flink.core.memory.MemorySegment import org.apache.flink.table.data._ import org.apache.flink.table.data.binary.BinaryRowDataUtil.BYTE_ARRAY_BASE_OFFSET import org.apache.flink.table.data.binary._ +import org.apache.flink.table.data.conversion.DataStructureConverters import org.apache.flink.table.data.util.DataFormatConverters import org.apache.flink.table.data.util.DataFormatConverters.IdentityConverter import org.apache.flink.table.functions.UserDefinedFunction @@ -42,6 +43,8 @@ import org.apache.flink.table.types.logical.LogicalTypeRoot._ import org.apache.flink.table.types.logical._ import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{getFieldCount, getPrecision, getScale, hasRoot} import org.apache.flink.table.types.logical.utils.LogicalTypeUtils.toInternalConversionClass +import org.apache.flink.table.types.utils.DataTypeUtils +import org.apache.flink.table.types.utils.DataTypeUtils.isInternal import org.apache.flink.types.{Row, RowKind} import scala.annotation.tailrec @@ -775,10 +778,119 @@ object CodeGenUtils { // -------------------------- Data Structure Conversion ------------------------------- + /** + * Generates code for converting the given term of external data type to an internal data + * structure. + * + * Use this function for converting at the edges of the API where primitive types CAN NOT occur + * and NO NULL CHECKING is required as it might have been done by surrounding layers. + */ + def genToInternalConverter( + ctx: CodeGeneratorContext, + sourceDataType: DataType) + : String => String = { + if (isInternal(sourceDataType)) { + externalTerm => s"$externalTerm" + } else { + val converter = DataStructureConverters.getConverter(sourceDataType) + val internalTypeTerm = boxedTypeTermForType(sourceDataType.getLogicalType) + val externalTypeTerm = typeTerm(sourceDataType.getConversionClass) + val converterTerm = ctx.addReusableConverter(converter) + externalTerm => + s"($internalTypeTerm) $converterTerm.toInternalOrNull(($externalTypeTerm) $externalTerm)" + } + } + + /** + * Generates code for converting the given term of external data type to an internal data + * structure. + * + * Use this function for converting at the edges of the API where PRIMITIVE TYPES can occur or + * the RESULT CAN BE NULL. + */ + def genToInternalConverterAll( + ctx: CodeGeneratorContext, + sourceDataType: DataType, + externalTerm: String) + : GeneratedExpression = { + val sourceType = sourceDataType.getLogicalType + val sourceClass = sourceDataType.getConversionClass + // convert external source type to internal structure + val internalResultTerm = if (isInternal(sourceDataType)) { + s"$externalTerm" + } else { + genToInternalConverter(ctx, sourceDataType)(externalTerm) + } + // extract null term from result term + if (sourceClass.isPrimitive) { + generateNonNullField(sourceType, internalResultTerm) + } else { + generateInputFieldUnboxing(ctx, sourceType, externalTerm, internalResultTerm) + } + } + + /** + * Generates code for converting the given term of internal data structure to the given + * external target data type. + * + * Use this function for converting at the edges of the API where primitive types CAN NOT occur + * and NO NULL CHECKING is required as it might have been done by surrounding layers. + */ + def genToExternalConverter( + ctx: CodeGeneratorContext, + targetDataType: DataType, + internalTerm: String) + : String = { + if (isInternal(targetDataType)) { + s"$internalTerm" + } else { + val converter = DataStructureConverters.getConverter(targetDataType) + val internalTypeTerm = boxedTypeTermForType(targetDataType.getLogicalType) + val externalTypeTerm = typeTerm(targetDataType.getConversionClass) + val converterTerm = ctx.addReusableConverter(converter) + s"($externalTypeTerm) $converterTerm.toExternal(($internalTypeTerm) $internalTerm)" + } + } + + /** + * Generates code for converting the given expression of internal data structure to the given + * external target data type. + * + * Use this function for converting at the edges of the API where PRIMITIVE TYPES can occur or + * the RESULT CAN BE NULL. + */ + def genToExternalConverterAll( + ctx: CodeGeneratorContext, + targetDataType: DataType, + internalExpr: GeneratedExpression) + : String = { + val targetType = targetDataType.getLogicalType + val targetTypeTerm = boxedTypeTermForType(targetType) + + // untyped null literal + if (hasRoot(internalExpr.resultType, NULL)) { + return s"($targetTypeTerm) null" + } + + // convert internal structure to target type + val externalResultTerm = if (isInternal(targetDataType)) { + s"($targetTypeTerm) ${internalExpr.resultTerm}" + } else { + genToExternalConverter(ctx, targetDataType, internalExpr.resultTerm) + } + // merge null term into the result term + if (targetDataType.getConversionClass.isPrimitive) { + externalResultTerm + } else { + s"${internalExpr.nullTerm} ? null : ($externalResultTerm)" + } + } + /** * If it's internally compatible, don't need to DataStructure converter. * clazz != classOf[Row] => Row can only infer GenericType[Row]. */ + @deprecated def isInternalClass(t: DataType): Boolean = { val clazz = t.getConversionClass clazz != classOf[Object] && clazz != classOf[Row] && @@ -786,10 +898,15 @@ object CodeGenUtils { clazz == toInternalConversionClass(fromDataTypeToLogicalType(t))) } + @deprecated private def isConverterIdentity(t: DataType): Boolean = { DataFormatConverters.getConverterForDataType(t).isInstanceOf[IdentityConverter[_]] } + /** + * @deprecated This uses the legacy [[DataFormatConverters]] including legacy types. + */ + @deprecated def genToInternal(ctx: CodeGeneratorContext, t: DataType, term: String): String = genToInternal(ctx, t)(term) @@ -798,7 +915,10 @@ object CodeGenUtils { * * Use this function for converting at the edges of the API where primitive types CAN NOT occur * and NO NULL CHECKING is required as it might have been done by surrounding layers. + * + * @deprecated This uses the legacy [[DataFormatConverters]] including legacy types. */ + @deprecated def genToInternal(ctx: CodeGeneratorContext, t: DataType): String => String = { if (isConverterIdentity(t)) { term => s"$term" @@ -813,11 +933,10 @@ object CodeGenUtils { } /** - * Generates code for converting the given external source data type to the internal data format. * - * Use this function for converting at the edges of the API where PRIMITIVE TYPES can occur or - * the RESULT CAN BE NULL. + * @deprecated This uses the legacy [[DataFormatConverters]] including legacy types. */ + @deprecated def genToInternalIfNeeded( ctx: CodeGeneratorContext, sourceDataType: DataType, @@ -839,6 +958,10 @@ object CodeGenUtils { } } + /** + * @deprecated This uses the legacy [[DataFormatConverters]] including legacy types. + */ + @deprecated def genToExternal( ctx: CodeGeneratorContext, targetType: DataType, @@ -856,10 +979,9 @@ object CodeGenUtils { } /** - * Generates code for converting the internal data format to the given external target data type. - * - * Use this function for converting at the edges of the API. + * @deprecated This uses the legacy [[DataFormatConverters]] including legacy types. */ + @deprecated def genToExternalIfNeeded( ctx: CodeGeneratorContext, targetDataType: DataType, diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGeneratorContext.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGeneratorContext.scala index d15d2e500f463..5ef90a2290d46 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGeneratorContext.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/CodeGeneratorContext.scala @@ -32,11 +32,11 @@ import org.apache.flink.table.runtime.util.collections._ import org.apache.flink.table.types.logical.LogicalTypeRoot._ import org.apache.flink.table.types.logical._ import org.apache.flink.util.InstantiationUtil - import org.apache.calcite.avatica.util.DateTimeUtils - import java.util.TimeZone +import org.apache.flink.table.data.conversion.DataStructureConverter + import scala.collection.mutable /** @@ -664,6 +664,33 @@ class CodeGeneratorContext(val tableConfig: TableConfig) { fieldTerm } + /** + * Adds a reusable [[DataStructureConverter]] to the member area of the generated class. + * + * @param converter converter to be added + * @param classLoaderTerm term to access the [[ClassLoader]] for user-defined classes + */ + def addReusableConverter( + converter: DataStructureConverter[_, _], + classLoaderTerm: String = null) + : String = { + + val converterTerm = addReusableObject(converter, "converter") + + val openConverter = if (classLoaderTerm != null) { + s""" + |$converterTerm.open($classLoaderTerm); + """.stripMargin + } else { + s""" + |$converterTerm.open(getRuntimeContext().getUserCodeClassLoader()); + """.stripMargin + } + reusableOpenStatements.add(openConverter) + + converterTerm + } + /** * Adds a reusable [[TypeSerializer]] to the member area of the generated class. * @@ -678,7 +705,7 @@ class CodeGeneratorContext(val tableConfig: TableConfig) { case None => val term = newName("typeSerializer") - val ser = InternalSerializers.create(t, new ExecutionConfig) + val ser = InternalSerializers.create(t) addReusableObjectInternal(ser, term, ser.getClass.getCanonicalName) reusableTypeSerializers(t) = term term diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala index 82a01224c4cee..46d151f7be03f 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala @@ -35,6 +35,8 @@ import org.apache.calcite.rex.{RexBuilder, RexExecutor, RexNode} import org.apache.calcite.sql.`type`.SqlTypeName import java.io.File +import org.apache.flink.table.data.conversion.DataStructureConverter + import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer @@ -275,4 +277,11 @@ class ConstantCodeGeneratorContext(tableConfig: TableConfig) runtimeContextTerm: String = null): String = { super.addReusableFunction(function, classOf[ConstantFunctionContext], "parameters") } + + override def addReusableConverter( + converter: DataStructureConverter[_, _], + classLoaderTerm: String = null) + : String = { + super.addReusableConverter(converter, "this.getClass().getClassLoader()") + } } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/BridgingSqlFunctionCallGen.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/BridgingSqlFunctionCallGen.scala index a15ac35e28ea6..abfaffbd50091 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/BridgingSqlFunctionCallGen.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/BridgingSqlFunctionCallGen.scala @@ -18,10 +18,14 @@ package org.apache.flink.table.planner.codegen.calls +import java.lang.reflect.Method +import java.util.Collections + +import org.apache.calcite.rex.{RexCall, RexCallBinding} import org.apache.flink.table.data.GenericRowData import org.apache.flink.table.functions.UserDefinedFunctionHelper.{SCALAR_EVAL, TABLE_EVAL} import org.apache.flink.table.functions.{FunctionKind, ScalarFunction, TableFunction, UserDefinedFunction} -import org.apache.flink.table.planner.codegen.CodeGenUtils.{genToExternalIfNeeded, genToInternalIfNeeded, newName, typeTerm} +import org.apache.flink.table.planner.codegen.CodeGenUtils._ import org.apache.flink.table.planner.codegen.GeneratedExpression.NEVER_NULL import org.apache.flink.table.planner.codegen._ import org.apache.flink.table.planner.functions.bridging.BridgingSqlFunction @@ -29,17 +33,13 @@ import org.apache.flink.table.planner.functions.inference.OperatorBindingCallCon import org.apache.flink.table.planner.utils.JavaScalaConversionUtil.toScala import org.apache.flink.table.runtime.collector.WrappingCollector import org.apache.flink.table.types.DataType +import org.apache.flink.table.types.extraction.ExtractionUtils import org.apache.flink.table.types.extraction.ExtractionUtils.{createMethodSignatureString, isAssignable, isInvokable, primitiveToWrapper} import org.apache.flink.table.types.inference.TypeInferenceUtil import org.apache.flink.table.types.logical.utils.LogicalTypeCasts.supportsAvoidingCast import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{hasRoot, isCompositeType} import org.apache.flink.table.types.logical.{LogicalType, LogicalTypeRoot, RowType} import org.apache.flink.util.Preconditions -import org.apache.calcite.rex.{RexCall, RexCallBinding} -import java.lang.reflect.Method -import java.util.Collections - -import org.apache.flink.table.types.extraction.ExtractionUtils /** * Generates a call to a user-defined [[ScalarFunction]] or [[TableFunction]]. @@ -110,15 +110,15 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { if (function.getDefinition.getKind == FunctionKind.TABLE) { Preconditions.checkState( - hasRoot(returnType, LogicalTypeRoot.ROW), - "Logical output type of function call should be a ROW type.", + isCompositeType(returnType), + "Logical output type of function call should be a composite type.", Seq(): _*) generateTableFunctionCall( ctx, functionTerm, externalOperands, outputDataType, - returnType.asInstanceOf[RowType]) + returnType) } else { generateScalarFunctionCall(ctx, functionTerm, externalOperands, outputDataType) } @@ -129,7 +129,7 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { functionTerm: String, externalOperands: Seq[GeneratedExpression], functionOutputDataType: DataType, - outputType: RowType) + outputType: LogicalType) : GeneratedExpression = { val resultCollectorTerm = generateResultCollector(ctx, functionOutputDataType, outputType) @@ -160,7 +160,7 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { def generateResultCollector( ctx: CodeGeneratorContext, outputDataType: DataType, - returnType: RowType) + returnType: LogicalType) : String = { val outputType = outputDataType.getLogicalType @@ -172,7 +172,7 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { val resultGenerator = new ExprCodeGenerator(collectorCtx, outputType.isNullable) .bindInput(outputType, externalResultTerm) val wrappedResult = resultGenerator.generateConverterResultExpression( - returnType, + returnType.asInstanceOf[RowType], classOf[GenericRowData]) s""" |${wrappedResult.code} @@ -193,7 +193,7 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { outputType, externalResultTerm, // nullability is handled by the expression code generator if necessary - CodeGenUtils.genToInternal(ctx, outputDataType), + genToInternalConverter(ctx, outputDataType), collectorCode) val resultCollectorTerm = newName("resultConverterCollector") CollectorCodeGenerator.addToContext(ctx, resultCollectorTerm, resultCollector) @@ -221,7 +221,7 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { s"($externalResultTypeTerm) (${typeTerm(externalResultClassBoxed)})" } val externalResultTerm = ctx.addReusableLocalVariable(externalResultTypeTerm, "externalResult") - val internalExpr = genToInternalIfNeeded(ctx, outputDataType, externalResultTerm) + val internalExpr = genToInternalConverterAll(ctx, outputDataType, externalResultTerm) // function call internalExpr.copy(code = @@ -241,7 +241,7 @@ class BridgingSqlFunctionCallGen(call: RexCall) extends CallGenerator { operands .zip(argumentDataTypes) .map { case (operand, dataType) => - operand.copy(resultTerm = genToExternalIfNeeded(ctx, dataType, operand)) + operand.copy(resultTerm = genToExternalConverterAll(ctx, dataType, operand)) } } diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayBooleanArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayBooleanArrayConverter.java index 39394139f4e40..e5eb43865ff2f 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayBooleanArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayBooleanArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code boolean[]} external type. */ @Internal -class ArrayBooleanArrayConverter implements DataStructureConverter { +public class ArrayBooleanArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayByteArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayByteArrayConverter.java index 6c8b665e70b9f..05baa77b4f0cc 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayByteArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayByteArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code byte[]} external type. */ @Internal -class ArrayByteArrayConverter implements DataStructureConverter { +public class ArrayByteArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayDoubleArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayDoubleArrayConverter.java index b442ff990646f..f0132718fbf5f 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayDoubleArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayDoubleArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code double[]} external type. */ @Internal -class ArrayDoubleArrayConverter implements DataStructureConverter { +public class ArrayDoubleArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayFloatArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayFloatArrayConverter.java index 3b8bf15652a7b..90be3de830484 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayFloatArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayFloatArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code float[]} external type. */ @Internal -class ArrayFloatArrayConverter implements DataStructureConverter { +public class ArrayFloatArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayIntArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayIntArrayConverter.java index fe8880a0fd606..56229652208cd 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayIntArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayIntArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code int[]} external type. */ @Internal -class ArrayIntArrayConverter implements DataStructureConverter { +public class ArrayIntArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayLongArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayLongArrayConverter.java index 963d14634141c..c495355adab88 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayLongArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayLongArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code long[]} external type. */ @Internal -class ArrayLongArrayConverter implements DataStructureConverter { +public class ArrayLongArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayObjectArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayObjectArrayConverter.java index 761d758064f9e..50490646fbf75 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayObjectArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayObjectArrayConverter.java @@ -39,7 +39,7 @@ */ @Internal @SuppressWarnings("unchecked") -class ArrayObjectArrayConverter implements DataStructureConverter { +public class ArrayObjectArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayShortArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayShortArrayConverter.java index 3b48ea400cf4f..7d536d84f8482 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayShortArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/ArrayShortArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link ArrayType} of {@code short[]} external type. */ @Internal -class ArrayShortArrayConverter implements DataStructureConverter { +public class ArrayShortArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateDateConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateDateConverter.java index e98089187407e..886591db4e449 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateDateConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateDateConverter.java @@ -26,7 +26,7 @@ * Converter for {@link DateType} of {@link java.sql.Date} external type. */ @Internal -class DateDateConverter implements DataStructureConverter { +public class DateDateConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateLocalDateConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateLocalDateConverter.java index 6df885de96504..d707d3969448f 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateLocalDateConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DateLocalDateConverter.java @@ -26,7 +26,7 @@ * Converter for {@link DateType} of {@link java.time.LocalDate} external type. */ @Internal -class DateLocalDateConverter implements DataStructureConverter { +public class DateLocalDateConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DayTimeIntervalDurationConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DayTimeIntervalDurationConverter.java index 1ea01c5ec49b9..d3da668b18853 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DayTimeIntervalDurationConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DayTimeIntervalDurationConverter.java @@ -27,7 +27,7 @@ * Converter for {@link DayTimeIntervalType} of {@link java.time.Duration} external type. */ @Internal -class DayTimeIntervalDurationConverter implements DataStructureConverter { +public class DayTimeIntervalDurationConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DecimalBigDecimalConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DecimalBigDecimalConverter.java index 850b6a0073867..76832a46242dc 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DecimalBigDecimalConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/DecimalBigDecimalConverter.java @@ -29,7 +29,7 @@ * Converter for {@link DecimalType} of {@link BigDecimal} external type. */ @Internal -class DecimalBigDecimalConverter implements DataStructureConverter { +public class DecimalBigDecimalConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/IdentityConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/IdentityConverter.java index 8d9c874c51b7d..d274ae8fc1f82 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/IdentityConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/IdentityConverter.java @@ -24,7 +24,7 @@ * No-op converter that just forwards its input. */ @Internal -class IdentityConverter implements DataStructureConverter { +public class IdentityConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampInstantConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampInstantConverter.java index fffefbe4e83d8..c8921b754c6cc 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampInstantConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampInstantConverter.java @@ -26,7 +26,7 @@ * Converter for {@link LocalZonedTimestampType} of {@link java.time.Instant} external type. */ @Internal -class LocalZonedTimestampInstantConverter implements DataStructureConverter { +public class LocalZonedTimestampInstantConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampIntConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampIntConverter.java index 349d34eb3467e..35fd54c1f8e5f 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampIntConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampIntConverter.java @@ -26,7 +26,7 @@ * Converter for {@link LocalZonedTimestampType} of {@link Integer} external type. */ @Internal -class LocalZonedTimestampIntConverter implements DataStructureConverter { +public class LocalZonedTimestampIntConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampLongConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampLongConverter.java index 6ddb7ebb0ff8f..0281a5185d095 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampLongConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/LocalZonedTimestampLongConverter.java @@ -26,7 +26,7 @@ * Converter for {@link LocalZonedTimestampType} of {@link Long} external type. */ @Internal -class LocalZonedTimestampLongConverter implements DataStructureConverter { +public class LocalZonedTimestampLongConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/MapMapConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/MapMapConverter.java index 24131b017e667..406036c22fa33 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/MapMapConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/MapMapConverter.java @@ -35,7 +35,7 @@ * Converter for {@link MapType}/{@link MultisetType} of {@link Map} external type. */ @Internal -class MapMapConverter implements DataStructureConverter> { +public class MapMapConverter implements DataStructureConverter> { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawByteArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawByteArrayConverter.java index db244073e3540..7ce245e33d544 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawByteArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawByteArrayConverter.java @@ -28,7 +28,7 @@ * Converter for {@link RawType} of {@code byte[]} external type. */ @Internal -class RawByteArrayConverter implements DataStructureConverter, byte[]> { +public class RawByteArrayConverter implements DataStructureConverter, byte[]> { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawObjectConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawObjectConverter.java index d93756aeb7c7c..db3bafb1aa7ac 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawObjectConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RawObjectConverter.java @@ -28,7 +28,7 @@ * Converter for {@link RawType} of object external type. */ @Internal -class RawObjectConverter implements DataStructureConverter, T> { +public class RawObjectConverter implements DataStructureConverter, T> { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java index c899ff39f0f15..b03197bd6f021 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/RowRowConverter.java @@ -32,7 +32,7 @@ * Converter for {@link RowType} of {@link Row} external type. */ @Internal -class RowRowConverter implements DataStructureConverter { +public class RowRowConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringByteArrayConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringByteArrayConverter.java index 3a7736cb479b4..94de7e683d946 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringByteArrayConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringByteArrayConverter.java @@ -27,7 +27,7 @@ * Converter for {@link CharType}/{@link VarCharType} of {@code byte[]} external type. */ @Internal -class StringByteArrayConverter implements DataStructureConverter { +public class StringByteArrayConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringStringConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringStringConverter.java index 290758bfb982b..1aed58bd76738 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringStringConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StringStringConverter.java @@ -27,7 +27,7 @@ * Converter for {@link CharType}/{@link VarCharType} of {@link String} external type. */ @Internal -class StringStringConverter implements DataStructureConverter { +public class StringStringConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StructuredObjectConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StructuredObjectConverter.java index 2d21db8606ddb..aa9ba954f611b 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StructuredObjectConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/StructuredObjectConverter.java @@ -46,7 +46,7 @@ */ @Internal @SuppressWarnings("unchecked") -class StructuredObjectConverter implements DataStructureConverter { +public class StructuredObjectConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLocalTimeConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLocalTimeConverter.java index d418f25f25aa3..77c6196b021cd 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLocalTimeConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLocalTimeConverter.java @@ -26,7 +26,7 @@ * Converter for {@link TimeType} of {@link java.time.LocalTime} external type. */ @Internal -class TimeLocalTimeConverter implements DataStructureConverter { +public class TimeLocalTimeConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLongConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLongConverter.java index 6cc79fb12b259..d84dcfccf8d9c 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLongConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeLongConverter.java @@ -25,7 +25,7 @@ * Converter for {@link TimeType} of {@link Long} external type. */ @Internal -class TimeLongConverter implements DataStructureConverter { +public class TimeLongConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeTimeConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeTimeConverter.java index 1c8b34a15446b..33293eac12b6e 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeTimeConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimeTimeConverter.java @@ -26,7 +26,7 @@ * Converter for {@link TimeType} of {@link java.sql.Time} external type. */ @Internal -class TimeTimeConverter implements DataStructureConverter { +public class TimeTimeConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampLocalDateTimeConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampLocalDateTimeConverter.java index c156715cb949c..bce2bf76eb197 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampLocalDateTimeConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampLocalDateTimeConverter.java @@ -26,7 +26,7 @@ * Converter for {@link TimestampType} of {@link java.time.LocalDateTime} external type. */ @Internal -class TimestampLocalDateTimeConverter implements DataStructureConverter { +public class TimestampLocalDateTimeConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampTimestampConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampTimestampConverter.java index f9a72f084bc8b..66a4fcbcd1787 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampTimestampConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/TimestampTimestampConverter.java @@ -26,7 +26,7 @@ * Converter for {@link TimestampType} of {@link java.sql.Timestamp} external type. */ @Internal -class TimestampTimestampConverter implements DataStructureConverter { +public class TimestampTimestampConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/YearMonthIntervalPeriodConverter.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/YearMonthIntervalPeriodConverter.java index 50f8d018ed034..ee871c8b46352 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/YearMonthIntervalPeriodConverter.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/data/conversion/YearMonthIntervalPeriodConverter.java @@ -30,7 +30,7 @@ * Converter for {@link YearMonthIntervalType} of {@link java.time.Period} external type. */ @Internal -class YearMonthIntervalPeriodConverter implements DataStructureConverter { +public class YearMonthIntervalPeriodConverter implements DataStructureConverter { private static final long serialVersionUID = 1L; From 41b17a8e75319cb265d9daf89a55e143eff11d10 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 18 May 2020 13:04:17 +0200 Subject: [PATCH 071/773] [hotfix][table-planner-blink] Fix exception for invalid function signature --- .../functions/inference/TypeInferenceOperandInference.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/inference/TypeInferenceOperandInference.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/inference/TypeInferenceOperandInference.java index e8d0aeea49ca7..75255b834453b 100644 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/inference/TypeInferenceOperandInference.java +++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/functions/inference/TypeInferenceOperandInference.java @@ -19,6 +19,7 @@ package org.apache.flink.table.planner.functions.inference; import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.functions.FunctionDefinition; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; @@ -69,6 +70,8 @@ public void inferOperandTypes(SqlCallBinding callBinding, RelDataType returnType returnType); try { inferOperandTypesOrError(unwrapTypeFactory(callBinding), callContext, operandTypes); + } catch (ValidationException e) { + // let operand checker fail } catch (Throwable t) { throw createUnexpectedException(callContext, t); } From d255bef5daca2aab238ed9e00ddf14a00193a5a4 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 18 May 2020 11:30:27 +0200 Subject: [PATCH 072/773] [FLINK-17541][table] Support inline structured types This enables inline structured types in the Blink planner. Inline structured types are extracted (e.g. in UDFs) and don't need to be registered in a catalog. This finalizes FLIP-65 for scalar and table functions because existing functions can be migrated with a replacement to the new type system. Structured type support should still be declared as experimental until we have more tests and can also deal with structured types in sources and sinks. This closes #12228. --- .../table/types/logical/StructuredType.java | 19 ++- .../types/logical/utils/LogicalTypeCasts.java | 2 +- .../plan/schema/StructuredRelDataType.java | 137 ++++++++++++++++++ .../planner/calcite/FlinkTypeFactory.scala | 10 ++ .../runtime/stream/sql/FunctionITCase.java | 106 ++++++++++++++ 5 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/schema/StructuredRelDataType.java diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/StructuredType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/StructuredType.java index d8aa593881a8d..2c934381c96d9 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/StructuredType.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/StructuredType.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; @@ -44,15 +45,14 @@ * by an {@link ObjectIdentifier} or anonymously defined, unregistered types (usually reflectively * extracted) that are identified by an implementation {@link Class}. * + *

Logical properties

+ * *

A structured type can declare a super type and allows single inheritance for more complex type * hierarchies, similar to JVM-based languages. * *

A structured type can be declared {@code final} for preventing further inheritance (default * behavior) or {@code not final} for allowing subtypes. * - *

A structured type must offer a default constructor with zero arguments or a full constructor - * that assigns all attributes. - * *

A structured type can be declared {@code not instantiable} if a more specific type is * required or {@code instantiable} if instances can be created from this type (default behavior). * @@ -61,6 +61,19 @@ * *

NOTE: Compared to the SQL standard, this class is incomplete. We might add new features such * as method declarations in the future. Also ordering is not supported yet. + * + *

Physical properties

+ * + *

A structured type can be defined fully logically (e.g. by using a {@code CREATE TYPE} DDL). The + * implementation class is optional and only used at the edges of the table ecosystem (e.g. when bridging + * to a function or connector). Serialization and equality ({@code hashCode/equals}) are handled by + * the runtime based on the logical type. In other words: {@code hashCode/equals} of an implementation + * class are not used. Custom equality, casting logic, and further overloaded operators will be supported + * once we allow defining methods on structured types. + * + *

An implementation class must offer a default constructor with zero arguments or a full constructor + * that assigns all attributes. Other physical properties such as the conversion classes of attributes + * are defined by a {@link DataType} when a structured type is used. */ @PublicEvolving public final class StructuredType extends UserDefinedType { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java index f22f9bee09711..ea63e0a382fed 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java @@ -305,7 +305,7 @@ private static boolean supportsCasting( } else if (targetRoot == DISTINCT_TYPE) { return supportsCasting(sourceType, ((DistinctType) targetType).getSourceType(), allowExplicit); } else if (sourceRoot == STRUCTURED_TYPE || targetRoot == STRUCTURED_TYPE) { - // TODO structured types are not supported yet + // inheritance is not supported yet, so structured type must be fully equal return false; } else if (sourceRoot == NULL) { // null can be cast to an arbitrary type diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/schema/StructuredRelDataType.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/schema/StructuredRelDataType.java new file mode 100644 index 0000000000000..4593a2866c836 --- /dev/null +++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/schema/StructuredRelDataType.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.schema; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; +import org.apache.flink.table.types.logical.StructuredType; +import org.apache.flink.table.types.logical.StructuredType.StructuredAttribute; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeComparability; +import org.apache.calcite.rel.type.RelDataTypeFamily; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.ObjectSqlType; +import org.apache.calcite.sql.type.SqlTypeName; + +import java.util.ArrayList; +import java.util.List; + +/** + * The {@link RelDataType} representation of a {@link StructuredType}. + * + *

It extends {@link ObjectSqlType} for preserving the original logical type (including an optional + * implementation class) and supporting anonymous/unregistered structured types from Table API. + */ +@Internal +public final class StructuredRelDataType extends ObjectSqlType { + + private final StructuredType structuredType; + + private StructuredRelDataType(StructuredType structuredType, List fields) { + super( + SqlTypeName.STRUCTURED, + createSqlIdentifier(structuredType), + structuredType.isNullable(), + fields, + createRelDataTypeComparability(structuredType)); + this.structuredType = structuredType; + computeDigest(); // recompute digest + } + + public static StructuredRelDataType create(FlinkTypeFactory factory, StructuredType structuredType) { + final List fields = new ArrayList<>(); + for (int i = 0; i < structuredType.getAttributes().size(); i++) { + final StructuredAttribute attribute = structuredType.getAttributes().get(i); + final RelDataTypeField field = new RelDataTypeFieldImpl( + attribute.getName(), + i, + factory.createFieldTypeFromLogicalType(attribute.getType())); + fields.add(field); + } + return new StructuredRelDataType(structuredType, fields); + } + + public StructuredType getStructuredType() { + return structuredType; + } + + public StructuredRelDataType createWithNullability(boolean nullable) { + if (nullable == isNullable()) { + return this; + } + return new StructuredRelDataType((StructuredType) structuredType.copy(nullable), fieldList); + } + + @Override + public RelDataTypeFamily getFamily() { + return this; // every user-defined type is its own family + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + // called by super constructor + if (structuredType == null) { + return; + } + if (withDetail) { + if (structuredType.getObjectIdentifier().isPresent()) { + sb.append(structuredType.asSerializableString()); + } + // in case of inline structured type we are using a temporary identifier + else { + sb.append(structuredType.asSummaryString()); + if (structuredType.isNullable()) { + sb.append(" NOT NULL"); + } + } + } else { + sb.append(structuredType.asSummaryString()); + } + } + + @Override + protected void computeDigest() { + final StringBuilder sb = new StringBuilder(); + generateTypeString(sb, true); + digest = sb.toString(); + } + + private static SqlIdentifier createSqlIdentifier(StructuredType structuredType) { + return structuredType.getObjectIdentifier() + .map(i -> new SqlIdentifier(i.toList(), SqlParserPos.ZERO)) + .orElseGet(() -> new SqlIdentifier(structuredType.asSummaryString(), SqlParserPos.ZERO)); + } + + private static RelDataTypeComparability createRelDataTypeComparability(StructuredType structuredType) { + switch (structuredType.getComparision()) { + case EQUALS: + return RelDataTypeComparability.UNORDERED; + case FULL: + return RelDataTypeComparability.ALL; + case NONE: + return RelDataTypeComparability.NONE; + default: + throw new IllegalArgumentException("Unsupported structured type comparision."); + } + } +} diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala index 61a0d0e8f5b53..57d64f86ad438 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala @@ -107,6 +107,10 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) // fields are not expanded in "SELECT *" StructKind.PEEK_FIELDS_NO_EXPAND) + case LogicalTypeRoot.STRUCTURED_TYPE => + val structuredType = t.asInstanceOf[StructuredType] + StructuredRelDataType.create(this, structuredType) + case LogicalTypeRoot.ARRAY => val arrayType = t.asInstanceOf[ArrayType] createArrayType(createFieldTypeFromLogicalType(arrayType.getElementType), -1) @@ -326,6 +330,9 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) case raw: RawRelDataType => raw.createWithNullability(isNullable) + case structured: StructuredRelDataType => + structured.createWithNullability(isNullable) + case generic: GenericRelDataType => new GenericRelDataType(generic.genericType, isNullable, typeSystem) @@ -521,6 +528,9 @@ object FlinkTypeFactory { case ROW if relDataType.isInstanceOf[RelRecordType] => toLogicalRowType(relDataType) + case STRUCTURED if relDataType.isInstanceOf[StructuredRelDataType] => + relDataType.asInstanceOf[StructuredRelDataType].getStructuredType + case MULTISET => new MultisetType(toLogicalType(relDataType.getComponentType)) case ARRAY => new ArrayType(toLogicalType(relDataType.getComponentType)) diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/FunctionITCase.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/FunctionITCase.java index 319e942b04426..c8929acf6ac75 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/FunctionITCase.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/FunctionITCase.java @@ -696,6 +696,37 @@ public void testRawLiteralScalarFunction() throws Exception { assertThat(TestCollectionTableFactory.getResult(), containsInAnyOrder(sinkData)); } + @Test + public void testStructuredScalarFunction() { + final List sourceData = Arrays.asList( + Row.of("Bob", 42), + Row.of("Alice", 12), + Row.of(null, 0) + ); + + final List sinkData = Arrays.asList( + Row.of("Bob 42", "Tyler"), + Row.of("Alice 12", "Tyler"), + Row.of("<>", "Tyler") + ); + + TestCollectionTableFactory.reset(); + TestCollectionTableFactory.initData(sourceData); + + tEnv().executeSql("CREATE TABLE SourceTable(s STRING, i INT NOT NULL) WITH ('connector' = 'COLLECTION')"); + tEnv().executeSql("CREATE TABLE SinkTable(s1 STRING, s2 STRING) WITH ('connector' = 'COLLECTION')"); + + tEnv().createTemporarySystemFunction("StructuredScalarFunction", StructuredScalarFunction.class); + execInsertSqlAndWaitResult( + "INSERT INTO SinkTable " + + "SELECT " + + " StructuredScalarFunction(StructuredScalarFunction(s, i)), " + + " StructuredScalarFunction('Tyler', 27).name " + + "FROM SourceTable"); + + assertThat(TestCollectionTableFactory.getResult(), equalTo(sinkData)); + } + @Test public void testInvalidCustomScalarFunction() { tEnv().executeSql("CREATE TABLE SinkTable(s STRING) WITH ('connector' = 'COLLECTION')"); @@ -744,6 +775,32 @@ public void testRowTableFunction() throws Exception { assertThat(TestCollectionTableFactory.getResult(), equalTo(sinkData)); } + @Test + public void testStructuredTableFunction() { + final List sourceData = Arrays.asList( + Row.of("Bob", 42), + Row.of("Alice", 12), + Row.of(null, 0) + ); + + final List sinkData = Arrays.asList( + Row.of("Bob", 42), + Row.of("Alice", 12), + Row.of(null, 0) + ); + + TestCollectionTableFactory.reset(); + TestCollectionTableFactory.initData(sourceData); + + tEnv().executeSql("CREATE TABLE SourceTable(s STRING, i INT NOT NULL) WITH ('connector' = 'COLLECTION')"); + tEnv().executeSql("CREATE TABLE SinkTable(s STRING, i INT NOT NULL) WITH ('connector' = 'COLLECTION')"); + + tEnv().createTemporarySystemFunction("StructuredTableFunction", StructuredTableFunction.class); + execInsertSqlAndWaitResult("INSERT INTO SinkTable SELECT t.name, t.age FROM SourceTable, LATERAL TABLE(StructuredTableFunction(s, i)) t"); + + assertThat(TestCollectionTableFactory.getResult(), equalTo(sinkData)); + } + @Test public void testDynamicTableFunction() throws Exception { final Row[] sinkData = new Row[]{ @@ -1008,4 +1065,53 @@ public TypeInference getTypeInference(DataTypeFactory typeFactory) { .build(); } } + + /** + * Function that creates and consumes structured types. + */ + public static class StructuredScalarFunction extends ScalarFunction { + public StructuredUser eval(String name, int age) { + if (name == null) { + return null; + } + return new StructuredUser(name, age); + } + + public String eval(StructuredUser user) { + if (user == null) { + return "<>"; + } + return user.toString(); + } + } + + /** + * Table function that returns a structured type. + */ + public static class StructuredTableFunction extends TableFunction { + public void eval(String name, int age) { + if (name == null) { + collect(null); + } + collect(new StructuredUser(name, age)); + } + } + + /** + * Example POJO for structured type. + */ + public static class StructuredUser { + public final String name; + public final int age; + + public StructuredUser(String name, int age) { + this.name = name; + this.age = age; + } + + @Override + public String toString() { + return name + " " + age; + } + } } From 1b0d7fddfb5e948fe9c6fa8b17f6c00569addd02 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 19 May 2020 14:45:56 +0200 Subject: [PATCH 073/773] [FLINK-17725][tests] Disable OkHttpClient timeouts for FileUploadHandlerTest In order to harden the test case FileUploadHandlerTest, this commit disables the timeouts of the used OkHttpClient. This closes #12248. --- .../runtime/rest/FileUploadHandlerTest.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java index 80fa4b957d8b0..74c0ee3021a98 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java @@ -40,6 +40,7 @@ import java.io.StringWriter; import java.lang.reflect.Field; import java.util.LinkedHashSet; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -130,7 +131,7 @@ private static MultipartBody.Builder addJsonPart(MultipartBody.Builder builder, @Test public void testUploadDirectoryRegeneration() throws Exception { - OkHttpClient client = new OkHttpClient(); + OkHttpClient client = createOkHttpClientWithNoTimeouts(); MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler(); @@ -146,7 +147,7 @@ public void testUploadDirectoryRegeneration() throws Exception { @Test public void testMixedMultipart() throws Exception { - OkHttpClient client = new OkHttpClient(); + OkHttpClient client = createOkHttpClientWithNoTimeouts(); MultipartUploadResource.MultipartMixedHandler mixedHandler = MULTIPART_UPLOAD_RESOURCE.getMixedHandler(); @@ -174,7 +175,7 @@ public void testMixedMultipart() throws Exception { @Test public void testJsonMultipart() throws Exception { - OkHttpClient client = new OkHttpClient(); + OkHttpClient client = createOkHttpClientWithNoTimeouts(); MultipartUploadResource.MultipartJsonHandler jsonHandler = MULTIPART_UPLOAD_RESOURCE.getJsonHandler(); @@ -202,7 +203,7 @@ public void testJsonMultipart() throws Exception { @Test public void testFileMultipart() throws Exception { - OkHttpClient client = new OkHttpClient(); + OkHttpClient client = createOkHttpClientWithNoTimeouts(); MultipartUploadResource.MultipartFileHandler fileHandler = MULTIPART_UPLOAD_RESOURCE.getFileHandler(); @@ -228,7 +229,7 @@ public void testFileMultipart() throws Exception { @Test public void testUploadCleanupOnUnknownAttribute() throws IOException { - OkHttpClient client = new OkHttpClient(); + OkHttpClient client = createOkHttpClientWithNoTimeouts(); Request request = buildMixedRequestWithUnknownAttribute(MULTIPART_UPLOAD_RESOURCE.getMixedHandler().getMessageHeaders().getTargetRestEndpointURL()); try (Response response = client.newCall(request).execute()) { @@ -244,7 +245,7 @@ public void testUploadCleanupOnUnknownAttribute() throws IOException { */ @Test public void testUploadCleanupOnFailure() throws IOException { - OkHttpClient client = new OkHttpClient(); + OkHttpClient client = createOkHttpClientWithNoTimeouts(); Request request = buildMalformedRequest(MULTIPART_UPLOAD_RESOURCE.getMixedHandler().getMessageHeaders().getTargetRestEndpointURL()); try (Response response = client.newCall(request).execute()) { @@ -256,6 +257,15 @@ public void testUploadCleanupOnFailure() throws IOException { verifyNoFileIsRegisteredToDeleteOnExitHook(); } + private OkHttpClient createOkHttpClientWithNoTimeouts() { + // don't fail if some OkHttpClient operations take longer. See FLINK-17725 + return new OkHttpClient.Builder() + .connectTimeout(0, TimeUnit.MILLISECONDS) + .writeTimeout(0, TimeUnit.MILLISECONDS) + .readTimeout(0, TimeUnit.MILLISECONDS) + .build(); + } + /** * DiskAttribute and DiskFileUpload class of netty store post chunks and file chunks as temp files on local disk. * By default, netty will register these temp files to java.io.DeleteOnExitHook which may lead to memory leak. From eead39dda060c9f91c145aadc57cc8177fa23242 Mon Sep 17 00:00:00 2001 From: Steve Whelan Date: Wed, 20 May 2020 05:31:45 -0400 Subject: [PATCH 074/773] [FLINK-16611][metrics][datadog] Send report in chunks --- docs/monitoring/metrics.md | 2 ++ docs/monitoring/metrics.zh.md | 2 ++ .../flink/metrics/datadog/DCounter.java | 1 + .../apache/flink/metrics/datadog/DMetric.java | 3 ++ .../apache/flink/metrics/datadog/DSeries.java | 4 +++ .../metrics/datadog/DatadogHttpReporter.java | 28 +++++++++++++------ 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md index 67cf3854601e9..8e532735975d3 100644 --- a/docs/monitoring/metrics.md +++ b/docs/monitoring/metrics.md @@ -770,6 +770,7 @@ Parameters: - `proxyHost` - (optional) The proxy host to use when sending to Datadog. - `proxyPort` - (optional) The proxy port to use when sending to Datadog, defaults to 8080. - `dataCenter` - (optional) The data center (`EU`/`US`) to connect to, defaults to `US`. +- `maxMetricsPerRequest` - (optional) The maximum number of metrics to include in each request, defaults to 2000. Example configuration: @@ -781,6 +782,7 @@ metrics.reporter.dghttp.tags: myflinkapp,prod metrics.reporter.dghttp.proxyHost: my.web.proxy.com metrics.reporter.dghttp.proxyPort: 8080 metrics.reporter.dhhttp.dataCenter: US +metrics.reporter.dhhttp.maxMetricsPerRequest: 2000 {% endhighlight %} diff --git a/docs/monitoring/metrics.zh.md b/docs/monitoring/metrics.zh.md index 8a59edf3d3caf..da249d81aa578 100644 --- a/docs/monitoring/metrics.zh.md +++ b/docs/monitoring/metrics.zh.md @@ -770,6 +770,7 @@ Parameters: - `proxyHost` - (optional) The proxy host to use when sending to Datadog. - `proxyPort` - (optional) The proxy port to use when sending to Datadog, defaults to 8080. - `dataCenter` - (optional) The data center (`EU`/`US`) to connect to, defaults to `US`. +- `maxMetricsPerRequest` - (optional) The maximum number of metrics to include in each request, defaults to 2000. Example configuration: @@ -781,6 +782,7 @@ metrics.reporter.dghttp.tags: myflinkapp,prod metrics.reporter.dghttp.proxyHost: my.web.proxy.com metrics.reporter.dghttp.proxyPort: 8080 metrics.reporter.dhhttp.dataCenter: US +metrics.reporter.dhhttp.maxMetricsPerRequest: 2000 {% endhighlight %} diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java index 549787cb9c8b7..14279233f8835 100644 --- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java +++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DCounter.java @@ -52,6 +52,7 @@ public Number getMetricValue() { return difference; } + @Override public void ackReport() { lastReportCount = currentReportCount; } diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java index 75a45251518c0..1b87435216800 100644 --- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java +++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DMetric.java @@ -78,4 +78,7 @@ public List> getPoints() { @JsonIgnore public abstract Number getMetricValue(); + + public void ackReport() { + } } diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java index 139d18930537f..fb631f28c788b 100644 --- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java +++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DSeries.java @@ -35,6 +35,10 @@ public DSeries() { series = new ArrayList<>(); } + public DSeries(List series) { + this.series = series; + } + public void addGauge(DGauge gauge) { series.add(gauge); } diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java index d32323bbc248d..7e569ebbcceaa 100644 --- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java +++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java @@ -56,6 +56,7 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled { private DatadogHttpClient client; private List configTags; + private int maxMetricsPerRequestValue; private final Clock clock = () -> System.currentTimeMillis() / 1000L; @@ -64,6 +65,7 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled { public static final String PROXY_PORT = "proxyPort"; public static final String DATA_CENTER = "dataCenter"; public static final String TAGS = "tags"; + public static final String MAX_METRICS_PER_REQUEST = "maxMetricsPerRequest"; @Override public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) { @@ -113,6 +115,7 @@ public void open(MetricConfig config) { String proxyHost = config.getString(PROXY_HOST, null); Integer proxyPort = config.getInteger(PROXY_PORT, 8080); String rawDataCenter = config.getString(DATA_CENTER, "US"); + maxMetricsPerRequestValue = config.getInteger(MAX_METRICS_PER_REQUEST, 2000); DataCenter dataCenter = DataCenter.valueOf(rawDataCenter); String tags = config.getString(TAGS, ""); @@ -120,7 +123,7 @@ public void open(MetricConfig config) { configTags = getTagsFromConfig(tags); - LOGGER.info("Configured DatadogHttpReporter with {tags={}, proxyHost={}, proxyPort={}, dataCenter={}", tags, proxyHost, proxyPort, dataCenter); + LOGGER.info("Configured DatadogHttpReporter with {tags={}, proxyHost={}, proxyPort={}, dataCenter={}, maxMetricsPerRequest={}", tags, proxyHost, proxyPort, dataCenter, maxMetricsPerRequestValue); } @Override @@ -137,14 +140,21 @@ public void report() { counters.values().forEach(request::addCounter); meters.values().forEach(request::addMeter); - try { - client.send(request); - counters.values().forEach(DCounter::ackReport); - LOGGER.debug("Reported series with size {}.", request.getSeries().size()); - } catch (SocketTimeoutException e) { - LOGGER.warn("Failed reporting metrics to Datadog because of socket timeout: {}", e.getMessage()); - } catch (Exception e) { - LOGGER.warn("Failed reporting metrics to Datadog.", e); + int totalMetrics = request.getSeries().size(); + int fromIndex = 0; + while (fromIndex < totalMetrics) { + int toIndex = Math.min(fromIndex + maxMetricsPerRequestValue, totalMetrics); + try { + DSeries chunk = new DSeries(request.getSeries().subList(fromIndex, toIndex)); + client.send(chunk); + chunk.getSeries().forEach(DMetric::ackReport); + LOGGER.debug("Reported series with size {}.", chunk.getSeries().size()); + } catch (SocketTimeoutException e) { + LOGGER.warn("Failed reporting metrics to Datadog because of socket timeout: {}", e.getMessage()); + } catch (Exception e) { + LOGGER.warn("Failed reporting metrics to Datadog.", e); + } + fromIndex = toIndex; } } From 6097d97a39877758d2729242186a19d86220e6ea Mon Sep 17 00:00:00 2001 From: Flavio Pompermaier Date: Wed, 20 May 2020 13:53:35 +0200 Subject: [PATCH 075/773] [FLINK-17622][connectors/jdbc] Remove useless switch for decimal in PostgresCatalog This closes #12090 --- .../jdbc/catalog/PostgresCatalog.java | 19 +++++++++++++++---- .../jdbc/catalog/PostgresCatalogITCase.java | 3 ++- .../jdbc/catalog/PostgresCatalogTestBase.java | 6 ++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java index c9b11246632db..31b4185f0069c 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java @@ -214,6 +214,14 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep } } + // Postgres jdbc driver maps several alias to real type, we use real type rather than alias: + // smallint <=> int2 + // integer <=> int4 + // int <=> int4 + // bigint <=> int8 + // float <=> float8 + // boolean <=> bool + // decimal <=> numeric public static final String PG_BYTEA = "bytea"; public static final String PG_BYTEA_ARRAY = "_bytea"; public static final String PG_SMALLINT = "int2"; @@ -224,8 +232,6 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep public static final String PG_BIGINT_ARRAY = "_int8"; public static final String PG_REAL = "float4"; public static final String PG_REAL_ARRAY = "_float4"; - public static final String PG_DECIMAL = "decimal"; - public static final String PG_DECIMAL_ARRAY = "_decimal"; public static final String PG_DOUBLE_PRECISION = "float8"; public static final String PG_DOUBLE_PRECISION_ARRAY = "_float8"; public static final String PG_NUMERIC = "numeric"; @@ -249,12 +255,19 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep public static final String PG_CHARACTER_VARYING = "varchar"; public static final String PG_CHARACTER_VARYING_ARRAY = "_varchar"; + /** + * Converts Postgres type to Flink {@link DataType}. + * + * @see org.postgresql.jdbc.TypeInfoCache + */ private DataType fromJDBCType(ResultSetMetaData metadata, int colIndex) throws SQLException { String pgType = metadata.getColumnTypeName(colIndex); int precision = metadata.getPrecision(colIndex); int scale = metadata.getScale(colIndex); + // pg types that gets replaced by jdbc driver: + // - decimal => numeric switch (pgType) { case PG_BOOLEAN: return DataTypes.BOOLEAN(); @@ -284,14 +297,12 @@ private DataType fromJDBCType(ResultSetMetaData metadata, int colIndex) throws S return DataTypes.DOUBLE(); case PG_DOUBLE_PRECISION_ARRAY: return DataTypes.ARRAY(DataTypes.DOUBLE()); - case PG_DECIMAL: case PG_NUMERIC: // see SPARK-26538: handle numeric without explicit precision and scale. if (precision > 0) { return DataTypes.DECIMAL(precision, metadata.getScale(colIndex)); } return DataTypes.DECIMAL(DecimalType.MAX_PRECISION, 18); - case PG_DECIMAL_ARRAY: case PG_NUMERIC_ARRAY: // see SPARK-26538: handle numeric without explicit precision and scale. if (precision > 0) { diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java index 422a81c2e6ef1..a5ad1ec463bc7 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java @@ -88,7 +88,7 @@ public void testPrimitiveTypes() throws Exception { List results = Lists.newArrayList( tEnv.sqlQuery(String.format("select * from %s", TABLE_PRIMITIVE_TYPE)).execute().collect()); - assertEquals("[1,[50],3,4,5.5,6.6,7.70000,true,a,b,c ,d,2016-06-22T19:10:25,2015-01-01,00:51:03,500.000000000000000000]", results.toString()); + assertEquals("[1,[50],3,4,5.5,6.6,7.70000,8.8,true,a,b,c ,d,2016-06-22T19:10:25,2015-01-01,00:51:03,500.000000000000000000]", results.toString()); } @Test @@ -107,6 +107,7 @@ public void testArrayTypes() throws Exception { "[6.6, 7.7, 8.8]," + "[7.70000, 8.80000, 9.90000]," + "[8.800000000000000000, 9.900000000000000000, 10.100000000000000000]," + + "[9.90, 10.10, 11.11]," + "[true, false, true]," + "[a, b, c]," + "[b, c, d]," + diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java index 62c0e1dc82631..b4b1b444851b1 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java @@ -164,6 +164,7 @@ public static TestTable getPrimitiveTable() { .field("real", DataTypes.FLOAT()) .field("double_precision", DataTypes.DOUBLE()) .field("numeric", DataTypes.DECIMAL(10, 5)) + .field("decimal", DataTypes.DECIMAL(10, 1)) .field("boolean", DataTypes.BOOLEAN()) .field("text", DataTypes.STRING()) .field("char", DataTypes.CHAR(1)) @@ -182,6 +183,7 @@ public static TestTable getPrimitiveTable() { "real real, " + "double_precision double precision, " + "numeric numeric(10, 5), " + + "decimal decimal(10, 1), " + "boolean boolean, " + "text text, " + "char char, " + @@ -199,6 +201,7 @@ public static TestTable getPrimitiveTable() { "5.5," + "6.6," + "7.7," + + "8.8," + "true," + "'a'," + "'b'," + @@ -224,6 +227,7 @@ public static TestTable getArrayTable() { .field("double_precision_arr", DataTypes.ARRAY(DataTypes.DOUBLE())) .field("numeric_arr", DataTypes.ARRAY(DataTypes.DECIMAL(10, 5))) .field("numeric_arr_default", DataTypes.ARRAY(DataTypes.DECIMAL(DecimalType.MAX_PRECISION, 18))) + .field("decimal_arr", DataTypes.ARRAY(DataTypes.DECIMAL(10, 2))) .field("boolean_arr", DataTypes.ARRAY(DataTypes.BOOLEAN())) .field("text_arr", DataTypes.ARRAY(DataTypes.STRING())) .field("char_arr", DataTypes.ARRAY(DataTypes.CHAR(1))) @@ -243,6 +247,7 @@ public static TestTable getArrayTable() { "double_precision_arr double precision[], " + "numeric_arr numeric(10, 5)[], " + "numeric_arr_default numeric[], " + + "decimal_arr decimal(10,2)[], " + "boolean_arr boolean[], " + "text_arr text[], " + "char_arr char[], " + @@ -261,6 +266,7 @@ public static TestTable getArrayTable() { "'{6.6,7.7,8.8}'," + "'{7.7,8.8,9.9}'," + "'{8.8,9.9,10.10}'," + + "'{9.9,10.10,11.11}'," + "'{true,false,true}'," + "'{a,b,c}'," + "'{b,c,d}'," + From 194b85b42749b03c5f1e79b5ae4377ab7230df36 Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Mon, 18 May 2020 19:51:03 +0200 Subject: [PATCH 076/773] [FLINK-15947] Fix table implicit conversions structure --- .../cassandra/CassandraConnectorITCase.java | 2 +- .../connector/hbase/HBaseConnectorITCase.java | 4 +- .../connectors/hive/HiveTableSinkTest.java | 2 +- .../connectors/hive/HiveTableSourceTest.java | 2 +- .../hive/HiveCatalogUseBlinkITCase.java | 2 +- .../table/catalog/hive/HiveTestUtils.java | 2 +- .../connector/jdbc/JdbcDataTypeTest.java | 2 +- .../table/JdbcDynamicTableSinkITCase.java | 2 +- .../table/JdbcDynamicTableSourceITCase.java | 2 +- .../jdbc/table/JdbcLookupTableITCase.java | 2 +- .../jdbc/table/JdbcTableSourceITCase.java | 2 +- .../jdbc/table/JdbcUpsertTableSinkITCase.java | 2 +- .../kafka/table/KafkaTableTestBase.java | 2 +- .../tests/BlinkStreamPythonUdfSqlJob.java | 2 +- .../tests/FlinkBatchPythonUdfSqlJob.java | 2 +- .../tests/FlinkStreamPythonUdfSqlJob.java | 2 +- .../flink/sql/tests/StreamSQLTestProgram.java | 2 +- .../table/examples/java/StreamSQLExample.java | 2 +- .../examples/java/StreamWindowSQLExample.java | 2 +- .../table/examples/java/WordCountSQL.java | 2 +- .../table/examples/java/WordCountTable.java | 2 +- .../examples/scala/StreamSQLExample.scala | 4 +- .../examples/scala/StreamTableExample.scala | 3 +- .../examples/scala/TPCHQuery3Table.scala | 3 +- .../table/examples/scala/WordCountSQL.scala | 3 +- .../table/examples/scala/WordCountTable.scala | 3 +- .../table/runtime/batch/AvroTypesITCase.java | 2 +- .../flink/orc/OrcTableSourceITCase.java | 2 +- .../parquet/ParquetTableSourceITCase.java | 2 +- .../apache/flink/ml/common/MLEnvironment.java | 4 +- .../flink/ml/pipeline/EstimatorBase.java | 2 +- .../flink/ml/pipeline/TransformerBase.java | 2 +- .../flink/ml/common/MLEnvironmentTest.java | 4 +- flink-python/pyflink/java_gateway.py | 1 + .../python/PythonFunctionFactoryTest.java | 4 +- .../PythonScalarFunctionOperatorTest.java | 2 +- .../PythonScalarFunctionOperatorTestBase.java | 2 +- ...wDataPythonScalarFunctionOperatorTest.java | 2 +- ...ArrowPythonScalarFunctionOperatorTest.java | 2 +- ...ArrowPythonScalarFunctionOperatorTest.java | 2 +- .../apache/flink/api/scala/FlinkILoop.scala | 10 +- .../flink/api/scala/ScalaShellITCase.scala | 6 +- .../gateway/local/ExecutionContext.java | 8 +- .../gateway/local/ExecutionContextTest.java | 2 +- .../java/BatchTableEnvironment.java | 4 +- .../java/StreamTableEnvironment.java | 4 +- .../internal/StreamTableEnvironmentImpl.java | 4 +- .../JavaDataStreamQueryOperation.java | 2 +- .../StreamTableEnvironmentImplTest.java | 2 +- .../scala/BatchTableEnvironment.scala | 5 +- .../scala/DataSetConversions.scala | 2 +- .../scala/DataStreamConversions.scala | 2 +- .../scala/StreamTableEnvironment.scala | 5 +- .../{ => bridge}/scala/TableConversions.scala | 2 +- .../internal/StreamTableEnvironmentImpl.scala | 9 +- .../api/{ => bridge}/scala/package.scala | 13 +- .../StreamTableEnvironmentImplTest.scala | 2 +- ...la => ImplicitExpressionConversions.scala} | 166 +--------------- .../api/ImplicitExpressionOperations.scala | 181 ++++++++++++++++++ .../org/apache/flink/table/api/package.scala | 8 +- .../api/ExpressionsConsistencyCheckTest.scala | 2 - .../delegation/BlinkExecutorFactory.java | 2 +- .../table/planner/expressions/package.scala | 29 --- .../plan/utils/WindowEmitStrategy.scala | 34 ++-- .../flink/table/api/EnvironmentTest.java | 2 +- .../table/planner/catalog/CatalogITCase.java | 2 +- .../table/api/TableEnvironmentITCase.scala | 4 +- .../table/api/TableEnvironmentTest.scala | 2 +- .../apache/flink/table/api/TableITCase.scala | 2 +- .../flink/table/api/batch/ExplainTest.scala | 4 +- .../flink/table/api/stream/ExplainTest.scala | 8 +- .../MatchRecognizeValidationTest.scala | 6 +- .../validation/OverWindowValidationTest.scala | 4 +- .../UserDefinedFunctionValidationTest.scala | 4 +- .../planner/codegen/agg/AggTestBase.scala | 2 +- .../planner/expressions/ArrayTypeTest.scala | 7 +- .../expressions/CompositeAccessTest.scala | 2 +- .../planner/expressions/DecimalTypeTest.scala | 3 +- .../planner/expressions/LiteralTest.scala | 2 +- .../planner/expressions/MapTypeTest.scala | 7 +- .../expressions/NonDeterministicTests.scala | 2 +- .../planner/expressions/RowTypeTest.scala | 4 +- .../expressions/ScalarFunctionsTest.scala | 4 +- .../expressions/TemporalTypesTest.scala | 5 +- .../UserDefinedScalarFunctionTest.scala | 5 +- .../utils/ExpressionTestBase.scala | 2 +- .../validation/ArrayTypeValidationTest.scala | 3 +- .../CompositeAccessValidationTest.scala | 3 +- .../validation/MapTypeValidationTest.scala | 3 +- .../validation/RowTypeValidationTest.scala | 3 +- .../ScalarFunctionsValidationTest.scala | 3 +- .../ScalarOperatorsValidationTest.scala | 3 +- .../match/PatternTranslatorTestBase.scala | 5 +- .../planner/plan/batch/sql/CalcTest.scala | 3 +- .../plan/batch/sql/DagOptimizationTest.scala | 7 +- .../plan/batch/sql/DeadlockBreakupTest.scala | 2 +- .../plan/batch/sql/LegacySinkTest.scala | 3 +- .../planner/plan/batch/sql/LimitTest.scala | 4 +- .../batch/sql/PartitionableSinkTest.scala | 3 +- .../planner/plan/batch/sql/RankTest.scala | 3 +- .../plan/batch/sql/SetOperatorsTest.scala | 3 +- .../plan/batch/sql/SortLimitTest.scala | 2 +- .../planner/plan/batch/sql/SortTest.scala | 2 +- .../plan/batch/sql/SubplanReuseTest.scala | 2 +- .../plan/batch/sql/TableScanTest.scala | 4 +- .../plan/batch/sql/TableSinkTest.scala | 2 +- .../planner/plan/batch/sql/UnionTest.scala | 2 +- .../batch/sql/agg/AggregateTestBase.scala | 3 +- .../plan/batch/sql/agg/GroupingSetsTest.scala | 3 +- .../batch/sql/agg/OverAggregateTest.scala | 3 +- .../batch/sql/agg/WindowAggregateTest.scala | 3 +- .../plan/batch/sql/join/JoinTestBase.scala | 3 +- .../plan/batch/sql/join/LookupJoinTest.scala | 2 +- .../batch/sql/join/SemiAntiJoinTestBase.scala | 3 +- .../batch/sql/join/SingleRowJoinTest.scala | 3 +- .../batch/sql/join/TemporalJoinTest.scala | 3 +- .../plan/batch/table/AggregateTest.scala | 2 +- .../planner/plan/batch/table/CalcTest.scala | 2 +- .../batch/table/ColumnFunctionsTest.scala | 2 +- .../plan/batch/table/CorrelateTest.scala | 3 +- .../plan/batch/table/GroupWindowTest.scala | 3 +- .../planner/plan/batch/table/JoinTest.scala | 5 +- .../plan/batch/table/PythonCalcTest.scala | 3 +- .../plan/batch/table/SetOperatorsTest.scala | 2 +- .../batch/table/TemporalTableJoinTest.scala | 3 +- .../AggregateStringExpressionTest.scala | 3 +- .../stringexpr/CalcStringExpressionTest.scala | 3 +- .../CorrelateStringExpressionTest.scala | 3 +- .../stringexpr/JoinStringExpressionTest.scala | 3 +- .../table/stringexpr/SetOperatorsTest.scala | 3 +- .../stringexpr/SortStringExpressionTest.scala | 3 +- .../validation/AggregateValidationTest.scala | 3 +- .../table/validation/CalcValidationTest.scala | 3 +- .../validation/CorrelateValidationTest.scala | 3 +- .../GroupWindowValidationTest.scala | 3 +- .../table/validation/JoinValidationTest.scala | 3 +- .../validation/OverWindowValidationTest.scala | 3 +- .../SetOperatorsValidationTest.scala | 3 +- .../table/validation/SortValidationTest.scala | 3 +- .../common/DistinctAggregateTestBase.scala | 4 +- .../planner/plan/common/UnnestTestBase.scala | 3 +- .../plan/common/ViewsExpandingTest.scala | 3 +- ...CalcPythonCorrelateTransposeRuleTest.scala | 5 +- .../logical/CalcRankTransposeRuleTest.scala | 2 +- .../logical/ConvertToNotInOrInRuleTest.scala | 2 +- .../DecomposeGroupingSetsRuleTest.scala | 3 +- .../ExpressionReductionRulesTest.scala | 3 +- .../FlinkAggregateJoinTransposeRuleTest.scala | 5 +- .../FlinkAggregateRemoveRuleTest.scala | 7 +- .../logical/FlinkCalcMergeRuleTest.scala | 2 +- .../logical/FlinkFilterJoinRuleTest.scala | 2 +- .../FlinkJoinPushExpressionsRuleTest.scala | 2 +- .../FlinkJoinToMultiJoinRuleTest.scala | 2 +- .../logical/FlinkLimit0RemoveRuleTest.scala | 2 +- ...kLogicalRankRuleForConstantRangeTest.scala | 2 +- .../FlinkLogicalRankRuleForRangeEndTest.scala | 3 +- .../logical/FlinkPruneEmptyRulesTest.scala | 2 +- ...kSemiAntiJoinFilterTransposeRuleTest.scala | 2 +- ...inkSemiAntiJoinJoinTransposeRuleTest.scala | 2 +- ...SemiAntiJoinProjectTransposeRuleTest.scala | 2 +- ...oinConditionEqualityTransferRuleTest.scala | 3 +- .../JoinConditionTypeCoerceRuleTest.scala | 2 +- ...DependentConditionDerivationRuleTest.scala | 2 +- ...ProjectSemiAntiJoinTransposeRuleTest.scala | 2 +- .../PruneAggregateCallRuleTestBase.scala | 3 +- .../logical/PythonCalcSplitRuleTest.scala | 5 +- .../PythonCorrelateSplitRuleTest.scala | 5 +- .../RankNumberColumnRemoveRuleTest.scala | 2 +- ...ReplaceIntersectWithSemiJoinRuleTest.scala | 2 +- .../ReplaceMinusWithAntiJoinRuleTest.scala | 2 +- .../logical/RewriteCoalesceRuleTest.scala | 3 +- .../logical/RewriteIntersectAllRuleTest.scala | 2 +- .../logical/RewriteMinusAllRuleTest.scala | 2 +- .../RewriteMultiJoinConditionRuleTest.scala | 4 +- .../SimplifyFilterConditionRuleTest.scala | 2 +- .../SimplifyJoinConditionRuleTest.scala | 2 +- .../logical/SplitAggregateRuleTest.scala | 2 +- ...PythonConditionFromCorrelateRuleTest.scala | 5 +- ...SplitPythonConditionFromJoinRuleTest.scala | 5 +- .../logical/WindowGroupReorderRuleTest.scala | 2 +- .../FlinkRewriteSubQueryRuleTest.scala | 2 +- .../subquery/SubQueryAntiJoinTest.scala | 2 +- .../subquery/SubQuerySemiJoinTest.scala | 3 +- ...ueryCorrelateVariablesValidationTest.scala | 3 +- .../batch/EnforceLocalAggRuleTestBase.scala | 2 +- .../RemoveRedundantLocalHashAggRuleTest.scala | 2 +- .../RemoveRedundantLocalRankRuleTest.scala | 2 +- .../RemoveRedundantLocalSortAggRuleTest.scala | 2 +- .../stream/ChangelogModeInferenceTest.scala | 3 +- .../planner/plan/stream/sql/CalcTest.scala | 3 +- .../plan/stream/sql/DagOptimizationTest.scala | 4 +- .../plan/stream/sql/DeduplicateTest.scala | 3 +- .../plan/stream/sql/LegacySinkTest.scala | 4 +- .../planner/plan/stream/sql/LimitTest.scala | 3 +- .../sql/MiniBatchIntervalInferTest.scala | 6 +- .../stream/sql/ModifiedMonotonicityTest.scala | 3 +- .../stream/sql/PartitionableSinkTest.scala | 3 +- .../planner/plan/stream/sql/RankTest.scala | 3 +- .../sql/RelTimeIndicatorConverterTest.scala | 2 +- .../plan/stream/sql/SetOperatorsTest.scala | 3 +- .../plan/stream/sql/SortLimitTest.scala | 3 +- .../planner/plan/stream/sql/SortTest.scala | 2 +- .../plan/stream/sql/SubplanReuseTest.scala | 2 +- .../plan/stream/sql/TableScanTest.scala | 3 +- .../plan/stream/sql/TableSinkTest.scala | 4 +- .../planner/plan/stream/sql/UnionTest.scala | 2 +- .../plan/stream/sql/agg/AggregateTest.scala | 3 +- .../sql/agg/DistinctAggregateTest.scala | 3 +- .../stream/sql/agg/GroupingSetsTest.scala | 4 +- .../stream/sql/agg/OverAggregateTest.scala | 3 +- .../sql/agg/TwoStageAggregateTest.scala | 2 +- .../stream/sql/agg/WindowAggregateTest.scala | 4 +- .../plan/stream/sql/join/JoinTest.scala | 3 +- .../plan/stream/sql/join/LookupJoinTest.scala | 2 +- .../stream/sql/join/SemiAntiJoinTest.scala | 2 +- .../stream/sql/join/TemporalJoinTest.scala | 3 +- .../plan/stream/sql/join/WindowJoinTest.scala | 3 +- .../plan/stream/table/AggregateTest.scala | 3 +- .../planner/plan/stream/table/CalcTest.scala | 3 +- .../stream/table/ColumnFunctionsTest.scala | 3 +- .../plan/stream/table/CorrelateTest.scala | 5 +- .../table/GroupWindowTableAggregateTest.scala | 4 +- .../plan/stream/table/GroupWindowTest.scala | 8 +- .../planner/plan/stream/table/JoinTest.scala | 2 +- .../stream/table/LegacyTableSourceTest.scala | 5 +- .../plan/stream/table/OverWindowTest.scala | 3 +- .../plan/stream/table/PythonCalcTest.scala | 3 +- .../plan/stream/table/SetOperatorsTest.scala | 2 +- .../stream/table/TableAggregateTest.scala | 3 +- .../plan/stream/table/TableSourceTest.scala | 3 +- .../stream/table/TemporalTableJoinTest.scala | 3 +- .../stream/table/TwoStageAggregateTest.scala | 3 +- .../AggregateStringExpressionTest.scala | 3 +- .../stringexpr/CalcStringExpressionTest.scala | 2 +- .../CorrelateStringExpressionTest.scala | 2 +- .../GroupWindowStringExpressionTest.scala | 3 +- ...owTableAggregateStringExpressionTest.scala | 4 +- .../OverWindowStringExpressionTest.scala | 3 +- .../SetOperatorsStringExpressionTest.scala | 3 +- .../TableAggregateStringExpressionTest.scala | 3 +- .../validation/AggregateValidationTest.scala | 3 +- .../table/validation/CalcValidationTest.scala | 3 +- .../validation/CorrelateValidationTest.scala | 2 +- ...upWindowTableAggregateValidationTest.scala | 4 +- .../GroupWindowValidationTest.scala | 3 +- .../LegacyTableSinkValidationTest.scala | 5 +- .../validation/OverWindowValidationTest.scala | 3 +- .../SetOperatorsValidationTest.scala | 5 +- .../TableAggregateValidationTest.scala | 3 +- .../TemporalTableJoinValidationTest.scala | 3 +- .../UnsupportedOpsValidationTest.scala | 5 +- .../plan/utils/FlinkRelOptUtilTest.scala | 4 +- .../batch/table/AggregationITCase.scala | 3 +- .../runtime/batch/table/CalcITCase.scala | 2 +- .../runtime/batch/table/CorrelateITCase.scala | 3 +- .../runtime/batch/table/DecimalITCase.scala | 3 +- .../batch/table/GroupWindowITCase.scala | 3 +- .../runtime/batch/table/JoinITCase.scala | 2 +- .../batch/table/LegacyTableSinkITCase.scala | 8 +- .../batch/table/OverWindowITCase.scala | 3 +- .../batch/table/SetOperatorsITCase.scala | 2 +- .../runtime/batch/table/SortITCase.scala | 3 +- .../runtime/batch/table/TableSinkITCase.scala | 3 +- .../harness/GroupAggregateHarnessTest.scala | 5 +- .../harness/OverWindowHarnessTest.scala | 7 +- .../harness/TableAggregateHarnessTest.scala | 5 +- .../runtime/stream/sql/AggregateITCase.scala | 9 +- .../stream/sql/AggregateRemoveITCase.scala | 3 +- .../stream/sql/AsyncLookupJoinITCase.scala | 2 +- .../runtime/stream/sql/CalcITCase.scala | 4 +- .../runtime/stream/sql/CorrelateITCase.scala | 9 +- .../stream/sql/DeduplicateITCase.scala | 3 +- .../runtime/stream/sql/JoinITCase.scala | 3 +- .../stream/sql/LegacyTableSourceITCase.scala | 2 +- .../stream/sql/Limit0RemoveITCase.scala | 4 +- .../runtime/stream/sql/LimitITCase.scala | 4 +- .../runtime/stream/sql/LookupJoinITCase.scala | 5 +- .../stream/sql/MatchRecognizeITCase.scala | 5 +- .../runtime/stream/sql/OverWindowITCase.scala | 3 +- .../stream/sql/PruneAggregateCallITCase.scala | 2 +- .../runtime/stream/sql/RankITCase.scala | 8 +- .../stream/sql/SemiAntiJoinStreamITCase.scala | 3 +- .../stream/sql/SetOperatorsITCase.scala | 3 +- .../runtime/stream/sql/SortITCase.scala | 4 +- .../runtime/stream/sql/SortLimitITCase.scala | 3 +- .../stream/sql/SplitAggregateITCase.scala | 4 +- .../sql/StreamFileSystemITCaseBase.scala | 2 +- .../sql/StreamTableEnvironmentITCase.scala | 5 +- .../runtime/stream/sql/TableScanITCase.scala | 11 +- .../stream/sql/TableSourceITCase.scala | 2 +- .../stream/sql/TemporalJoinITCase.scala | 4 +- .../stream/sql/TemporalSortITCase.scala | 3 +- .../stream/sql/TimeAttributeITCase.scala | 5 +- .../runtime/stream/sql/TimestampITCase.scala | 5 +- .../runtime/stream/sql/UnnestITCase.scala | 7 +- .../runtime/stream/sql/ValuesITCase.scala | 3 +- .../stream/sql/WindowAggregateITCase.scala | 10 +- .../runtime/stream/sql/WindowJoinITCase.scala | 3 +- .../stream/table/AggregateITCase.scala | 5 +- .../runtime/stream/table/CalcITCase.scala | 5 +- .../stream/table/CorrelateITCase.scala | 6 +- .../stream/table/GroupWindowITCase.scala | 4 +- .../GroupWindowTableAggregateITCase.scala | 11 +- .../runtime/stream/table/JoinITCase.scala | 7 +- .../stream/table/LegacyTableSinkITCase.scala | 9 +- .../table/MiniBatchGroupWindowITCase.scala | 4 +- .../stream/table/OverWindowITCase.scala | 4 +- .../stream/table/RetractionITCase.scala | 3 +- .../stream/table/SetOperatorsITCase.scala | 3 +- .../runtime/stream/table/SubQueryITCase.scala | 3 +- .../stream/table/TableAggregateITCase.scala | 5 +- .../stream/table/TableSinkITCase.scala | 5 +- .../runtime/utils/StreamTableEnvUtil.scala | 2 +- .../runtime/utils/StreamingTestBase.scala | 5 +- .../utils/StreamingWithStateTestBase.scala | 2 +- .../table/planner/utils/TableTestBase.scala | 8 +- .../api/{ => bridge}/java/package-info.java | 12 +- .../table/executor/StreamExecutorFactory.java | 2 +- .../internal/BatchTableEnvironmentImpl.scala | 6 +- .../internal/BatchTableEnvironmentImpl.scala | 6 +- .../PlannerExpressionParserImpl.scala | 4 +- .../flink/table/expressions/package.scala | 4 +- .../table/util/DummyExecutionEnvironment.java | 2 +- .../table/api/StreamTableEnvironmentTest.java | 2 +- .../table/catalog/PathResolutionTest.java | 2 +- .../table/catalog/ViewExpansionTest.java | 2 +- .../runtime/batch/JavaTableSourceITCase.java | 2 +- .../runtime/batch/sql/GroupingSetsITCase.java | 2 +- .../runtime/batch/sql/JavaSqlITCase.java | 2 +- .../table/JavaTableEnvironmentITCase.java | 2 +- .../runtime/stream/sql/FunctionITCase.java | 2 +- .../runtime/stream/sql/JavaSqlITCase.java | 2 +- .../runtime/stream/table/FunctionITCase.java | 2 +- .../runtime/stream/table/ValuesITCase.java | 2 +- .../table/api/TableEnvironmentITCase.scala | 14 +- .../table/api/TableEnvironmentTest.scala | 11 +- .../apache/flink/table/api/TableITCase.scala | 7 +- .../flink/table/api/TableSourceTest.scala | 11 +- .../api/batch/BatchTableEnvironmentTest.scala | 5 +- .../flink/table/api/batch/ExplainTest.scala | 7 +- .../table/api/batch/sql/AggregateTest.scala | 3 +- .../flink/table/api/batch/sql/CalcTest.scala | 5 +- .../table/api/batch/sql/CorrelateTest.scala | 5 +- .../api/batch/sql/DistinctAggregateTest.scala | 3 +- .../table/api/batch/sql/GroupWindowTest.scala | 7 +- .../api/batch/sql/GroupingSetsTest.scala | 3 +- .../flink/table/api/batch/sql/JoinTest.scala | 3 +- .../api/batch/sql/SetOperatorsTest.scala | 3 +- .../api/batch/sql/SingleRowJoinTest.scala | 5 +- .../api/batch/sql/TemporalTableJoinTest.scala | 9 +- .../sql/validation/CalcValidationTest.scala | 4 +- .../validation/CorrelateValidationTest.scala | 4 +- .../GroupWindowValidationTest.scala | 8 +- .../validation/InsertIntoValidationTest.scala | 4 +- .../sql/validation/JoinValidationTest.scala | 5 +- .../validation/OverWindowValidationTest.scala | 8 +- .../sql/validation/SortValidationTest.scala | 5 +- .../table/api/batch/table/AggregateTest.scala | 5 +- .../table/api/batch/table/CalcTest.scala | 3 +- .../api/batch/table/ColumnFunctionsTest.scala | 4 +- .../table/api/batch/table/CorrelateTest.scala | 8 +- .../api/batch/table/GroupWindowTest.scala | 8 +- .../table/api/batch/table/JoinTest.scala | 3 +- .../api/batch/table/SetOperatorsTest.scala | 3 +- .../batch/table/TemporalTableJoinTest.scala | 8 +- .../AggregateStringExpressionTest.scala | 5 +- .../stringexpr/CalcStringExpressionTest.scala | 8 +- .../CorrelateStringExpressionTest.scala | 3 +- .../stringexpr/JoinStringExpressionTest.scala | 4 +- .../table/stringexpr/SetOperatorsTest.scala | 7 +- .../stringexpr/SortStringExpressionTest.scala | 3 +- .../validation/AggregateValidationTest.scala | 4 +- .../table/validation/CalcValidationTest.scala | 5 +- .../validation/CorrelateValidationTest.scala | 4 +- .../GroupWindowValidationTest.scala | 4 +- .../validation/InsertIntoValidationTest.scala | 4 +- .../table/validation/JoinValidationTest.scala | 5 +- .../validation/OverWindowValidationTest.scala | 4 +- .../SetOperatorsValidationTest.scala | 5 +- .../table/validation/SortValidationTest.scala | 4 +- .../flink/table/api/stream/ExplainTest.scala | 5 +- .../stream/StreamTableEnvironmentTest.scala | 10 +- ...StreamTableEnvironmentValidationTest.scala | 7 +- .../table/api/stream/sql/AggregateTest.scala | 7 +- .../table/api/stream/sql/CorrelateTest.scala | 6 +- .../stream/sql/DistinctAggregateTest.scala | 3 +- .../api/stream/sql/GroupWindowTest.scala | 3 +- .../flink/table/api/stream/sql/JoinTest.scala | 8 +- .../api/stream/sql/MatchRecognizeTest.scala | 3 +- .../table/api/stream/sql/OverWindowTest.scala | 3 +- .../api/stream/sql/SetOperatorsTest.scala | 3 +- .../flink/table/api/stream/sql/SortTest.scala | 3 +- .../stream/sql/TemporalTableJoinTest.scala | 8 +- .../table/api/stream/sql/UnionTest.scala | 5 +- .../validation/CorrelateValidationTest.scala | 4 +- .../validation/InsertIntoValidationTest.scala | 2 +- .../sql/validation/JoinValidationTest.scala | 4 +- .../MatchRecognizeValidationTest.scala | 4 +- .../validation/OverWindowValidationTest.scala | 5 +- .../sql/validation/SortValidationTest.scala | 4 +- .../WindowAggregateValidationTest.scala | 4 +- .../api/stream/table/AggregateTest.scala | 6 +- .../table/api/stream/table/CalcTest.scala | 3 +- .../stream/table/ColumnFunctionsTest.scala | 3 +- .../api/stream/table/CorrelateTest.scala | 8 +- .../table/GroupWindowTableAggregateTest.scala | 6 +- .../api/stream/table/GroupWindowTest.scala | 4 +- .../table/api/stream/table/JoinTest.scala | 3 +- .../api/stream/table/OverWindowTest.scala | 6 +- .../api/stream/table/SetOperatorsTest.scala | 3 +- .../api/stream/table/TableAggregateTest.scala | 5 +- .../api/stream/table/TableSourceTest.scala | 4 +- .../stream/table/TemporalTableJoinTest.scala | 3 +- .../AggregateStringExpressionTest.scala | 4 +- .../stringexpr/CalcStringExpressionTest.scala | 3 +- .../CorrelateStringExpressionTest.scala | 2 +- .../GroupWindowStringExpressionTest.scala | 6 +- ...owTableAggregateStringExpressionTest.scala | 4 +- .../OverWindowStringExpressionTest.scala | 7 +- .../SetOperatorsStringExpressionTest.scala | 3 +- .../TableAggregateStringExpressionTest.scala | 3 +- .../validation/AggregateValidationTest.scala | 4 +- .../table/validation/CalcValidationTest.scala | 8 +- .../validation/CorrelateValidationTest.scala | 2 +- ...upWindowTableAggregateValidationTest.scala | 4 +- .../GroupWindowValidationTest.scala | 4 +- .../validation/InsertIntoValidationTest.scala | 5 +- .../table/validation/JoinValidationTest.scala | 4 +- .../validation/OverWindowValidationTest.scala | 4 +- .../SetOperatorsValidationTest.scala | 4 +- .../TableAggregateValidationTest.scala | 8 +- .../validation/TableSinkValidationTest.scala | 5 +- .../TableSourceValidationTest.scala | 4 +- .../TemporalTableJoinValidationTest.scala | 8 +- .../UnsupportedOpsValidationTest.scala | 4 +- .../ColumnFunctionsValidationTest.scala | 4 +- .../InlineTableValidationTest.scala | 4 +- .../TableEnvironmentValidationTest.scala | 4 +- .../validation/TableSinksValidationTest.scala | 4 +- .../TableSourceValidationTest.scala | 7 +- .../UserDefinedFunctionValidationTest.scala | 4 +- .../table/catalog/CatalogTableITCase.scala | 2 +- .../table/expressions/ArrayTypeTest.scala | 8 +- .../expressions/CompositeAccessTest.scala | 3 +- .../expressions/DateTimeFunctionTest.scala | 7 +- .../table/expressions/DecimalTypeTest.scala | 6 +- .../flink/table/expressions/LiteralTest.scala | 4 +- .../flink/table/expressions/MapTypeTest.scala | 8 +- .../expressions/NonDeterministicTests.scala | 5 +- .../flink/table/expressions/RowTypeTest.scala | 8 +- .../expressions/ScalarFunctionsTest.scala | 4 +- .../expressions/ScalarOperatorsTest.scala | 4 +- .../table/expressions/TemporalTypesTest.scala | 10 +- .../UserDefinedScalarFunctionTest.scala | 14 +- .../utils/ExpressionTestBase.scala | 4 +- .../validation/ArrayTypeValidationTest.scala | 4 +- .../CompositeAccessValidationTest.scala | 4 +- .../validation/MapTypeValidationTest.scala | 4 +- .../validation/RowTypeValidationTest.scala | 4 +- .../ScalarFunctionsValidationTest.scala | 6 +- .../ScalarOperatorsValidationTest.scala | 4 +- .../match/MatchRecognizeValidationTest.scala | 3 +- .../match/PatternTranslatorTestBase.scala | 6 +- ...CalcPythonCorrelateTransposeRuleTest.scala | 3 +- .../plan/ExpressionReductionRulesTest.scala | 5 +- .../table/plan/NormalizationRulesTest.scala | 8 +- .../table/plan/PythonCalcSplitRuleTest.scala | 5 +- .../plan/PythonCorrelateSplitRuleTest.scala | 3 +- .../table/plan/QueryDecorrelationTest.scala | 3 +- .../table/plan/RetractionRulesTest.scala | 6 +- ...PythonConditionFromCorrelateRuleTest.scala | 3 +- ...SplitPythonConditionFromJoinRuleTest.scala | 3 +- .../plan/TimeIndicatorConversionTest.scala | 8 +- .../table/plan/UpdatingPlanCheckerTest.scala | 4 +- .../runtime/batch/sql/AggregateITCase.scala | 5 +- .../table/runtime/batch/sql/CalcITCase.scala | 13 +- .../table/runtime/batch/sql/JoinITCase.scala | 9 +- .../batch/sql/PartitionableSinkITCase.scala | 2 +- .../batch/sql/SetOperatorsITCase.scala | 4 +- .../table/runtime/batch/sql/SortITCase.scala | 5 +- .../batch/sql/TableEnvironmentITCase.scala | 10 +- .../runtime/batch/sql/TableSourceITCase.scala | 5 +- .../runtime/batch/table/AggregateITCase.scala | 11 +- .../runtime/batch/table/CalcITCase.scala | 12 +- .../runtime/batch/table/CorrelateITCase.scala | 9 +- .../batch/table/GroupWindowITCase.scala | 9 +- .../runtime/batch/table/JoinITCase.scala | 13 +- .../batch/table/SetOperatorsITCase.scala | 4 +- .../runtime/batch/table/SortITCase.scala | 6 +- .../batch/table/TableEnvironmentITCase.scala | 8 +- .../runtime/batch/table/TableITCase.scala | 4 +- .../runtime/batch/table/TableSinkITCase.scala | 11 +- .../batch/table/TableSourceITCase.scala | 13 +- .../harness/AggFunctionHarnessTest.scala | 9 +- .../harness/GroupAggregateHarnessTest.scala | 6 +- .../runtime/harness/MatchHarnessTest.scala | 9 +- .../harness/TableAggregateHarnessTest.scala | 6 +- .../runtime/stream/TimeAttributesITCase.scala | 9 +- .../runtime/stream/sql/InsertIntoITCase.scala | 5 +- .../table/runtime/stream/sql/JoinITCase.scala | 7 +- .../stream/sql/MatchRecognizeITCase.scala | 9 +- .../runtime/stream/sql/OverWindowITCase.scala | 4 +- .../stream/sql/SetOperatorsITCase.scala | 4 +- .../table/runtime/stream/sql/SortITCase.scala | 7 +- .../table/runtime/stream/sql/SqlITCase.scala | 5 +- .../stream/sql/TableSourceITCase.scala | 3 +- .../stream/sql/TemporalJoinITCase.scala | 7 +- .../stream/table/AggregateITCase.scala | 5 +- .../runtime/stream/table/CalcITCase.scala | 4 +- .../stream/table/CorrelateITCase.scala | 7 +- .../stream/table/GroupWindowITCase.scala | 4 +- .../GroupWindowTableAggregateITCase.scala | 6 +- .../runtime/stream/table/JoinITCase.scala | 5 +- .../stream/table/OverWindowITCase.scala | 8 +- .../stream/table/RetractionITCase.scala | 4 +- .../stream/table/SetOperatorsITCase.scala | 4 +- .../stream/table/TableAggregateITCase.scala | 6 +- .../stream/table/TableSinkITCase.scala | 9 +- .../stream/table/TableSourceITCase.scala | 12 +- .../flink/table/utils/TableTestBase.scala | 8 +- .../src/main/java/SpendReport.java | 2 +- .../src/main/scala/SpendReport.scala | 1 - 522 files changed, 1306 insertions(+), 1251 deletions(-) rename flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/{ => bridge}/java/BatchTableEnvironment.java (99%) rename flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/{ => bridge}/java/StreamTableEnvironment.java (99%) rename flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/{ => bridge}/java/internal/StreamTableEnvironmentImpl.java (99%) rename flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/{ => bridge}/java/internal/StreamTableEnvironmentImplTest.java (98%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/BatchTableEnvironment.scala (98%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/DataSetConversions.scala (97%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/DataStreamConversions.scala (98%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/StreamTableEnvironment.scala (99%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/TableConversions.scala (98%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/internal/StreamTableEnvironmentImpl.scala (97%) rename flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/package.scala (86%) rename flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/{ => bridge}/scala/internal/StreamTableEnvironmentImplTest.scala (98%) rename flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/{expressionDsl.scala => ImplicitExpressionConversions.scala} (84%) create mode 100644 flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionOperations.scala delete mode 100644 flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/expressions/package.scala rename flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/{ => bridge}/java/package-info.java (77%) rename flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/{ => bridge}/java/internal/BatchTableEnvironmentImpl.scala (96%) rename flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/{ => bridge}/scala/internal/BatchTableEnvironmentImpl.scala (94%) diff --git a/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java b/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java index c14e94d00f80e..d06a48bb677aa 100644 --- a/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java +++ b/flink-connectors/flink-connector-cassandra/src/test/java/org/apache/flink/streaming/connectors/cassandra/CassandraConnectorITCase.java @@ -45,8 +45,8 @@ import org.apache.flink.streaming.api.functions.sink.SinkContextUtil; import org.apache.flink.streaming.runtime.operators.WriteAheadSinkTestBase; import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.testutils.junit.FailsOnJava11; import org.apache.flink.types.Row; diff --git a/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseConnectorITCase.java b/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseConnectorITCase.java index f04d5fd6cb2a8..2a1ffa26b0df7 100644 --- a/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseConnectorITCase.java +++ b/flink-connectors/flink-connector-hbase/src/test/java/org/apache/flink/connector/hbase/HBaseConnectorITCase.java @@ -36,10 +36,10 @@ import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; import org.apache.flink.table.api.internal.TableImpl; -import org.apache.flink.table.api.java.BatchTableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.descriptors.DescriptorProperties; import org.apache.flink.table.factories.TableFactoryService; import org.apache.flink.table.functions.ScalarFunction; diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java index b823a9bef0fdf..d2682462237eb 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java @@ -32,7 +32,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.CatalogTableImpl; import org.apache.flink.table.catalog.ObjectPath; diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java index 55bf470aa9448..3c05d70df3ba6 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java @@ -31,9 +31,9 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.table.api.internal.TableEnvironmentImpl; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.catalog.CatalogPartitionSpec; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.ObjectPath; diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogUseBlinkITCase.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogUseBlinkITCase.java index d0a0fc689aaa0..d3c6ed467cb9e 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogUseBlinkITCase.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogUseBlinkITCase.java @@ -28,7 +28,7 @@ import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.Types; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.CatalogFunctionImpl; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.CatalogTableBuilder; diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveTestUtils.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveTestUtils.java index 6ded44d30c6ef..3519a588c78f5 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveTestUtils.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveTestUtils.java @@ -21,7 +21,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.CatalogTest; import org.apache.flink.table.catalog.exceptions.CatalogException; import org.apache.flink.table.catalog.hive.client.HiveShimLoader; diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/JdbcDataTypeTest.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/JdbcDataTypeTest.java index f5654b4865d74..c1a40cd159dba 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/JdbcDataTypeTest.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/JdbcDataTypeTest.java @@ -21,7 +21,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.ValidationException; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.junit.Assert; import org.junit.Test; diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSinkITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSinkITCase.java index a2f7f77f21bbf..7288a019e9971 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSinkITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSinkITCase.java @@ -28,7 +28,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java index 48be89e6c0b5a..ca05816d7e473 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcDynamicTableSourceITCase.java @@ -21,7 +21,7 @@ import org.apache.flink.connector.jdbc.JdbcTestBase; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.planner.runtime.utils.StreamTestSink; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java index 793ea9d8f1234..783ebdf7723af 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcLookupTableITCase.java @@ -26,7 +26,7 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.types.DataType; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java index 81156961a6e9e..102110d1d3bdd 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcTableSourceITCase.java @@ -22,7 +22,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcUpsertTableSinkITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcUpsertTableSinkITCase.java index 69a6ac97f1f92..a23b7f97b0ca1 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcUpsertTableSinkITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/table/JdbcUpsertTableSinkITCase.java @@ -28,7 +28,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaTableTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaTableTestBase.java index 7ffe26f2ae900..3a459650dfe2b 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaTableTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/table/KafkaTableTestBase.java @@ -24,7 +24,7 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.connectors.kafka.KafkaTestBase; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.data.RowData; import org.apache.flink.table.planner.runtime.utils.TableEnvUtil; diff --git a/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/BlinkStreamPythonUdfSqlJob.java b/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/BlinkStreamPythonUdfSqlJob.java index c254b422c55b3..583c2e9b375cc 100644 --- a/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/BlinkStreamPythonUdfSqlJob.java +++ b/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/BlinkStreamPythonUdfSqlJob.java @@ -19,7 +19,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import java.util.ArrayList; diff --git a/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkBatchPythonUdfSqlJob.java b/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkBatchPythonUdfSqlJob.java index c5d05ed746781..af4bd51350b21 100644 --- a/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkBatchPythonUdfSqlJob.java +++ b/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkBatchPythonUdfSqlJob.java @@ -18,7 +18,7 @@ package org.apache.flink.python.tests; import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.types.Row; import java.util.ArrayList; diff --git a/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkStreamPythonUdfSqlJob.java b/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkStreamPythonUdfSqlJob.java index e509e23b476a4..4dace3515d8a3 100644 --- a/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkStreamPythonUdfSqlJob.java +++ b/flink-end-to-end-tests/flink-python-test/src/main/java/org/apache/flink/python/tests/FlinkStreamPythonUdfSqlJob.java @@ -19,7 +19,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import java.util.ArrayList; diff --git a/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java b/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java index 074b615931032..21f001aa8054d 100644 --- a/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java +++ b/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java @@ -46,8 +46,8 @@ import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.sources.DefinedFieldMapping; import org.apache.flink.table.sources.DefinedRowtimeAttributes; import org.apache.flink.table.sources.RowtimeAttributeDescriptor; diff --git a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java index 85de3881599bf..7c71a44c8b42d 100644 --- a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java +++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java @@ -23,7 +23,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import java.util.Arrays; import java.util.Objects; diff --git a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamWindowSQLExample.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamWindowSQLExample.java index b75815090c1e7..0e8d2db48cfbf 100644 --- a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamWindowSQLExample.java +++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamWindowSQLExample.java @@ -20,7 +20,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.apache.flink.util.FileUtils; diff --git a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java index 570683495546f..1851fffb2182f 100644 --- a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java +++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountSQL.java @@ -21,7 +21,7 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import static org.apache.flink.table.api.Expressions.$; diff --git a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java index 0d4157280ec40..385da2ecda1ab 100644 --- a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java +++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/WordCountTable.java @@ -21,7 +21,7 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import static org.apache.flink.table.api.Expressions.$; diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala index cf57c70de6a1b..f9795146a6f3b 100644 --- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala +++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.examples.scala import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ /** * Simple example for demonstrating the use of SQL on a Stream Table in Scala. diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala index a4b39aa8e9bd9..41b71a3ae95f1 100644 --- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala +++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamTableExample.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.examples.scala import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ /** * Simple example for demonstrating the use of Table API on a Stream Table. diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala index 4b5b930d7b196..f6314df318b40 100644 --- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala +++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/TPCHQuery3Table.scala @@ -18,7 +18,8 @@ package org.apache.flink.table.examples.scala import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ /** * This program implements a modified version of the TPC-H query 3. The diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala index 5a13d4034a61a..533caa6674dd7 100644 --- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala +++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountSQL.scala @@ -18,7 +18,8 @@ package org.apache.flink.table.examples.scala import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ /** * Simple example that shows how the Batch SQL API is used in Scala. diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala index 0879733443e56..a869d4d201c34 100644 --- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala +++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/WordCountTable.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.examples.scala import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ /** * Simple example for demonstrating the use of the Table API for a Word Count in Scala. diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/table/runtime/batch/AvroTypesITCase.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/table/runtime/batch/AvroTypesITCase.java index b76080a26f18a..7dfe9e38fbc86 100644 --- a/flink-formats/flink-avro/src/test/java/org/apache/flink/table/runtime/batch/AvroTypesITCase.java +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/table/runtime/batch/AvroTypesITCase.java @@ -28,7 +28,7 @@ import org.apache.flink.formats.avro.generated.User; import org.apache.flink.formats.avro.utils.AvroKryoSerializerUtils; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase; import org.apache.flink.test.util.TestBaseUtils; import org.apache.flink.types.Row; diff --git a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceITCase.java b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceITCase.java index b9c972d25c098..6890d7e2f0886 100644 --- a/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceITCase.java +++ b/flink-formats/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceITCase.java @@ -21,8 +21,8 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.BatchTableEnvironment; import org.apache.flink.test.util.MultipleProgramsTestBase; import org.apache.flink.types.Row; diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/ParquetTableSourceITCase.java b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/ParquetTableSourceITCase.java index 3d50ad47aaf81..8e7c43bd3282b 100644 --- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/ParquetTableSourceITCase.java +++ b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/ParquetTableSourceITCase.java @@ -23,8 +23,8 @@ import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.utils.TestUtil; import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.BatchTableEnvironment; import org.apache.flink.test.util.MultipleProgramsTestBase; import org.apache.flink.types.Row; diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java index 595ac2c423271..aa45042c729f2 100644 --- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java @@ -22,8 +22,8 @@ import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.BatchTableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; /** * The MLEnvironment stores the necessary context in Flink. diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java index 3b6bcc143d71a..35d5e695ff7bf 100644 --- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java @@ -27,8 +27,8 @@ import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.internal.TableImpl; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.util.Preconditions; /** diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java index ed3c374cbd90b..0ad369b5c6ee8 100644 --- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java @@ -27,7 +27,7 @@ import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.util.Preconditions; /** diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java index 50f87c5acba20..e9633c5273705 100644 --- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java +++ b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java @@ -22,8 +22,8 @@ import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.BatchTableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.junit.Assert; import org.junit.Test; diff --git a/flink-python/pyflink/java_gateway.py b/flink-python/pyflink/java_gateway.py index b778515a2bd22..0f8b9049ea0a4 100644 --- a/flink-python/pyflink/java_gateway.py +++ b/flink-python/pyflink/java_gateway.py @@ -130,6 +130,7 @@ def import_flink_view(gateway): # Import the classes used by PyFlink java_import(gateway.jvm, "org.apache.flink.table.api.*") java_import(gateway.jvm, "org.apache.flink.table.api.java.*") + java_import(gateway.jvm, "org.apache.flink.table.api.bridge.java.*") java_import(gateway.jvm, "org.apache.flink.table.api.dataview.*") java_import(gateway.jvm, "org.apache.flink.table.catalog.*") java_import(gateway.jvm, "org.apache.flink.table.descriptors.*") diff --git a/flink-python/src/test/java/org/apache/flink/client/python/PythonFunctionFactoryTest.java b/flink-python/src/test/java/org/apache/flink/client/python/PythonFunctionFactoryTest.java index 4acdb0a3af594..6e407398169e0 100644 --- a/flink-python/src/test/java/org/apache/flink/client/python/PythonFunctionFactoryTest.java +++ b/flink-python/src/test/java/org/apache/flink/client/python/PythonFunctionFactoryTest.java @@ -22,8 +22,8 @@ import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.BatchTableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.util.FileUtils; import java.io.File; diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTest.java index aad8a3fc702cc..85529881c04c4 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTest.java @@ -25,7 +25,7 @@ import org.apache.flink.python.env.PythonEnvironmentManager; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.util.TestHarnessUtil; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.runtime.types.CRow; import org.apache.flink.table.runtime.typeutils.PythonTypeUtils; diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTestBase.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTestBase.java index fc88cee2ed51c..05cdc63a09659 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTestBase.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/PythonScalarFunctionOperatorTestBase.java @@ -31,7 +31,7 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction; import org.apache.flink.table.runtime.runners.python.scalar.AbstractPythonScalarFunctionRunnerTest; diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/RowDataPythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/RowDataPythonScalarFunctionOperatorTest.java index 252448577586c..d45148f4038df 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/RowDataPythonScalarFunctionOperatorTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/RowDataPythonScalarFunctionOperatorTest.java @@ -27,7 +27,7 @@ import org.apache.flink.python.env.PythonEnvironmentManager; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.data.RowData; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.runtime.typeutils.PythonTypeUtils; diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java index c6f5515db4ca0..62139ed3076fd 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/ArrowPythonScalarFunctionOperatorTest.java @@ -24,7 +24,7 @@ import org.apache.flink.python.env.PythonEnvironmentManager; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.util.TestHarnessUtil; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.runtime.arrow.ArrowUtils; import org.apache.flink.table.runtime.arrow.ArrowWriter; diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/RowDataArrowPythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/RowDataArrowPythonScalarFunctionOperatorTest.java index e1742c5fd0766..706a32f98848a 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/RowDataArrowPythonScalarFunctionOperatorTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/scalar/arrow/RowDataArrowPythonScalarFunctionOperatorTest.java @@ -27,7 +27,7 @@ import org.apache.flink.python.env.PythonEnvironmentManager; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.data.RowData; import org.apache.flink.table.functions.python.PythonFunctionInfo; import org.apache.flink.table.runtime.arrow.ArrowUtils; diff --git a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala index 6368589c6f0c4..a36bf97564aff 100644 --- a/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala +++ b/flink-scala-shell/src/main/scala/org/apache/flink/api/scala/FlinkILoop.scala @@ -18,14 +18,15 @@ package org.apache.flink.api.scala -import java.io.{BufferedReader, File, FileOutputStream} import org.apache.flink.api.java.{JarHelper, ScalaShellEnvironment, ScalaShellStreamEnvironment} -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala.{BatchTableEnvironment, StreamTableEnvironment} +import org.apache.flink.table.api.bridge.scala.{BatchTableEnvironment, StreamTableEnvironment} import org.apache.flink.util.AbstractID +import java.io.{BufferedReader, File, FileOutputStream} + import scala.tools.nsc.interpreter._ @@ -139,7 +140,8 @@ class FlinkILoop( "org.apache.flink.api.scala.utils._", "org.apache.flink.streaming.api.scala._", "org.apache.flink.streaming.api.windowing.time._", - "org.apache.flink.table.api.scala._", + "org.apache.flink.table.api._", + "org.apache.flink.table.api.bridge.scala._", "org.apache.flink.types.Row" ) diff --git a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala index f49a2aa50a053..d4c987a30564e 100644 --- a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala +++ b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala @@ -432,8 +432,10 @@ class ScalaShellITCase extends TestLogger { """.stripMargin val output = processInShell(input) - Assert.assertTrue(output.contains("error: object util is not a member of package org.apache." + - "flink.table.api.java")) + Assert.assertTrue(output.contains("the java list size is: 5")) + Assert.assertFalse(output.toLowerCase.contains("failed")) + Assert.assertFalse(output.toLowerCase.contains("error")) + Assert.assertFalse(output.toLowerCase.contains("exception")) } @Test diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java index 4396ba96b9d63..4f5e41859f3e2 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java @@ -39,11 +39,11 @@ import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.internal.BatchTableEnvironmentImpl; +import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.BatchTableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.api.java.internal.BatchTableEnvironmentImpl; -import org.apache.flink.table.api.java.internal.StreamTableEnvironmentImpl; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.CatalogManager; import org.apache.flink.table.catalog.CatalogTableImpl; diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java index f44513e8787d8..f038395c9f7f8 100644 --- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java +++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java @@ -27,9 +27,9 @@ import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.table.api.config.OptimizerConfigOptions; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.GenericInMemoryCatalog; import org.apache.flink.table.catalog.hive.HiveCatalog; diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/BatchTableEnvironment.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/BatchTableEnvironment.java similarity index 99% rename from flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/BatchTableEnvironment.java rename to flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/BatchTableEnvironment.java index 507bb09b9139c..bf972f072a572 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/BatchTableEnvironment.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/BatchTableEnvironment.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.api.java; +package org.apache.flink.table.api.bridge.java; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -502,7 +502,7 @@ static BatchTableEnvironment create(ExecutionEnvironment executionEnvironment, T .executionConfig(executionEnvironment.getConfig()) .build(); - Class clazz = Class.forName("org.apache.flink.table.api.java.internal.BatchTableEnvironmentImpl"); + Class clazz = Class.forName("org.apache.flink.table.api.bridge.java.internal.BatchTableEnvironmentImpl"); Constructor con = clazz.getConstructor( ExecutionEnvironment.class, TableConfig.class, diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/StreamTableEnvironment.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/StreamTableEnvironment.java similarity index 99% rename from flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/StreamTableEnvironment.java rename to flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/StreamTableEnvironment.java index 22d10b9a0af0d..b9d611e80f080 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/StreamTableEnvironment.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/StreamTableEnvironment.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.api.java; +package org.apache.flink.table.api.bridge.java; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.JobExecutionResult; @@ -28,7 +28,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.internal.StreamTableEnvironmentImpl; +import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; import org.apache.flink.table.descriptors.ConnectorDescriptor; import org.apache.flink.table.descriptors.StreamTableDescriptor; import org.apache.flink.table.expressions.Expression; diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java similarity index 99% rename from flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java rename to flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java index 5abbc7f18661a..15877ad52c96c 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.api.java.internal; +package org.apache.flink.table.api.bridge.java.internal; import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeinfo.TypeInformation; @@ -34,8 +34,8 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.Types; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentImpl; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.catalog.CatalogManager; import org.apache.flink.table.catalog.FunctionCatalog; import org.apache.flink.table.catalog.GenericInMemoryCatalog; diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/operations/JavaDataStreamQueryOperation.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/operations/JavaDataStreamQueryOperation.java index 9bd294a015016..77655c29967a5 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/operations/JavaDataStreamQueryOperation.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/operations/JavaDataStreamQueryOperation.java @@ -21,7 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.expressions.Expression; diff --git a/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImplTest.java b/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java similarity index 98% rename from flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImplTest.java rename to flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java index 620c4c4c02fe5..954fc45678b84 100644 --- a/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImplTest.java +++ b/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.api.java.internal; +package org.apache.flink.table.api.bridge.java.internal; import org.apache.flink.api.common.time.Time; import org.apache.flink.api.dag.Transformation; diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/BatchTableEnvironment.scala similarity index 98% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/BatchTableEnvironment.scala index 4d595dc880036..a769a56c4e134 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/BatchTableEnvironment.scala @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.scala +package org.apache.flink.table.api.bridge.scala import org.apache.flink.api.common.JobExecutionResult import org.apache.flink.api.common.typeinfo.TypeInformation @@ -26,7 +26,6 @@ import org.apache.flink.table.descriptors.{BatchTableDescriptor, ConnectorDescri import org.apache.flink.table.expressions.Expression import org.apache.flink.table.functions.{AggregateFunction, TableFunction} import org.apache.flink.table.module.ModuleManager -import org.apache.flink.table.sinks.TableSink /** * The [[TableEnvironment]] for a Scala batch [[ExecutionEnvironment]] that works @@ -391,7 +390,7 @@ object BatchTableEnvironment { .build val clazz = Class - .forName("org.apache.flink.table.api.scala.internal.BatchTableEnvironmentImpl") + .forName("org.apache.flink.table.api.bridge.scala.internal.BatchTableEnvironmentImpl") val con = clazz .getConstructor( classOf[ExecutionEnvironment], diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/DataSetConversions.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/DataSetConversions.scala similarity index 97% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/DataSetConversions.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/DataSetConversions.scala index 4d80e7554a699..c51e36721bc81 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/DataSetConversions.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/DataSetConversions.scala @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.scala +package org.apache.flink.table.api.bridge.scala import org.apache.flink.annotation.PublicEvolving import org.apache.flink.api.common.typeinfo.TypeInformation diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/DataStreamConversions.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/DataStreamConversions.scala similarity index 98% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/DataStreamConversions.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/DataStreamConversions.scala index 1360f5cd56372..4ccc1b4f3176c 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/DataStreamConversions.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/DataStreamConversions.scala @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.scala +package org.apache.flink.table.api.bridge.scala import org.apache.flink.annotation.PublicEvolving import org.apache.flink.api.common.typeinfo.TypeInformation diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/StreamTableEnvironment.scala similarity index 99% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/StreamTableEnvironment.scala index 3b9f0eea5dd8c..dd0373e539565 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/StreamTableEnvironment.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/StreamTableEnvironment.scala @@ -15,18 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.scala +package org.apache.flink.table.api.bridge.scala import org.apache.flink.annotation.PublicEvolving import org.apache.flink.api.common.JobExecutionResult import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.api.{TableEnvironment, _} import org.apache.flink.table.descriptors.{ConnectorDescriptor, StreamTableDescriptor} import org.apache.flink.table.expressions.Expression import org.apache.flink.table.functions.{AggregateFunction, TableAggregateFunction, TableFunction} -import org.apache.flink.table.sinks.TableSink /** * This table environment is the entry point and central context for creating Table and SQL diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/TableConversions.scala similarity index 98% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/TableConversions.scala index 65ffd7054e724..592b3636db35f 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/TableConversions.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/TableConversions.scala @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.api.scala +package org.apache.flink.table.api.bridge.scala import org.apache.flink.annotation.PublicEvolving import org.apache.flink.api.common.typeinfo.TypeInformation diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala similarity index 97% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala index 6cd9d2c68b3b8..7c99e37f97968 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImpl.scala @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.scala.internal +package org.apache.flink.table.api.bridge.scala.internal import org.apache.flink.annotation.Internal -import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.dag.Transformation import org.apache.flink.api.scala._ @@ -27,8 +26,8 @@ import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream} import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JStreamExecutionEnvironment} import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala.StreamTableEnvironment import org.apache.flink.table.catalog.{CatalogManager, FunctionCatalog, GenericInMemoryCatalog, ObjectIdentifier} import org.apache.flink.table.delegation.{Executor, ExecutorFactory, Planner, PlannerFactory} import org.apache.flink.table.descriptors.{ConnectorDescriptor, StreamTableDescriptor} @@ -44,7 +43,7 @@ import org.apache.flink.table.typeutils.FieldInfoUtils import java.util import java.util.{Collections, List => JList, Map => JMap} -import _root_.scala.collection.JavaConverters._ +import scala.collection.JavaConverters._ /** * The implementation for a Scala [[StreamTableEnvironment]]. This enables conversions from/to @@ -68,7 +67,7 @@ class StreamTableEnvironmentImpl ( functionCatalog, planner, isStreaming) - with org.apache.flink.table.api.scala.StreamTableEnvironment { + with org.apache.flink.table.api.bridge.scala.StreamTableEnvironment { override def fromDataStream[T](dataStream: DataStream[T]): Table = { val queryOperation = asQueryOperation(dataStream, None) diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/package.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/package.scala similarity index 86% rename from flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/package.scala rename to flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/package.scala index 556c657df8881..6e390543d2aae 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/package.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/bridge/scala/package.scala @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api +package org.apache.flink.table.api.bridge import org.apache.flink.api.scala.{DataSet, _} import org.apache.flink.streaming.api.scala.DataStream import org.apache.flink.table.api.internal.TableImpl -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.{ImplicitExpressionConversions, ImplicitExpressionOperations, Table, ValidationException} import org.apache.flink.types.Row import _root_.scala.language.implicitConversions @@ -37,7 +37,7 @@ import _root_.scala.language.implicitConversions * * {{{ * import org.apache.flink.table.api._ - * import org.apache.flink.table.api.scala._ + * import org.apache.flink.table.api.bridge.scala._ * }}} * * More information about the entry points of the API can be found in [[StreamTableEnvironment]]. @@ -50,12 +50,7 @@ import _root_.scala.language.implicitConversions * Please refer to the website documentation about how to construct and run table programs that are * connected to the DataStream API. */ -package object scala extends ImplicitExpressionConversions { - - // This package object should not extend from ImplicitExpressionConversions but would clash with - // "org.apache.flink.table.api._" therefore we postpone splitting the package object into - // two and let users update there imports first. All users should import both `api._` and - // `api.scala._`. +package object scala { implicit def tableConversions(table: Table): TableConversions = { new TableConversions(table.asInstanceOf[TableImpl]) diff --git a/flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImplTest.scala b/flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImplTest.scala similarity index 98% rename from flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImplTest.scala rename to flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImplTest.scala index f8f5a0c6ae9a9..0f84d9b65daed 100644 --- a/flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImplTest.scala +++ b/flink-table/flink-table-api-scala-bridge/src/test/scala/org/apache/flink/table/api/bridge/scala/internal/StreamTableEnvironmentImplTest.scala @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.api.scala.internal +package org.apache.flink.table.api.bridge.scala.internal import java.util.{Collections, List => JList} diff --git a/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/expressionDsl.scala b/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionConversions.scala similarity index 84% rename from flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/expressionDsl.scala rename to flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionConversions.scala index a1239a95fc9d9..540d23351b1c9 100644 --- a/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/expressionDsl.scala +++ b/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionConversions.scala @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.table.api import org.apache.flink.annotation.PublicEvolving import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.table.api.internal.BaseExpressions -import org.apache.flink.table.expressions.ApiExpressionUtils._ -import org.apache.flink.table.expressions._ -import org.apache.flink.table.functions.BuiltInFunctionDefinitions._ +import org.apache.flink.table.expressions.ApiExpressionUtils.{unresolvedCall, unresolvedRef, valueLiteral} +import org.apache.flink.table.expressions.{ApiExpressionUtils, Expression, TableSymbol, TimePointUnit} +import org.apache.flink.table.functions.BuiltInFunctionDefinitions.{DISTINCT, RANGE_TO} import org.apache.flink.table.functions.{ScalarFunction, TableFunction, UserDefinedAggregateFunction, UserDefinedFunctionHelper, _} import org.apache.flink.table.types.DataType import org.apache.flink.types.Row @@ -33,163 +33,7 @@ import java.sql.{Date, Time, Timestamp} import java.time.{LocalDate, LocalDateTime, LocalTime} import java.util.{List => JList, Map => JMap} -import _root_.scala.language.implicitConversions - -/** - * These are all the operations that can be used to construct an [[Expression]] AST for - * expression operations. - */ -@PublicEvolving -trait ImplicitExpressionOperations extends BaseExpressions[Expression, Expression] { - private[flink] def expr: Expression - - override def toExpr: Expression = expr - - override protected def toApiSpecificExpression(expression: Expression): Expression = expression - - /** - * Specifies a name for an expression i.e. a field. - * - * @param name name for one field - * @param extraNames additional names if the expression expands to multiple fields - * @return field with an alias - */ - def as(name: Symbol, extraNames: Symbol*): Expression = as(name.name, extraNames.map(_.name): _*) - - /** - * Boolean AND in three-valued logic. - */ - def && (other: Expression): Expression = and(other) - - /** - * Boolean OR in three-valued logic. - */ - def || (other: Expression): Expression = or(other) - - /** - * Greater than. - */ - def > (other: Expression): Expression = isGreater(other) - - /** - * Greater than or equal. - */ - def >= (other: Expression): Expression = isGreaterOrEqual(other) - - /** - * Less than. - */ - def < (other: Expression): Expression = isLess(other) - - /** - * Less than or equal. - */ - def <= (other: Expression): Expression = isLessOrEqual(other) - - /** - * Equals. - */ - def === (other: Expression): Expression = isEqual(other) - - /** - * Not equal. - */ - def !== (other: Expression): Expression = isNotEqual(other) - - /** - * Whether boolean expression is not true; returns null if boolean is null. - */ - def unary_! : Expression = unresolvedCall(NOT, expr) - - /** - * Returns negative numeric. - */ - def unary_- : Expression = Expressions.negative(expr) - - /** - * Returns numeric. - */ - def unary_+ : Expression = expr - - /** - * Returns left plus right. - */ - def + (other: Expression): Expression = plus(other) - - /** - * Returns left minus right. - */ - def - (other: Expression): Expression = minus(other) - - /** - * Returns left divided by right. - */ - def / (other: Expression): Expression = dividedBy(other) - - /** - * Returns left multiplied by right. - */ - def * (other: Expression): Expression = times(other) - - /** - * Returns the remainder (modulus) of left divided by right. - * The result is negative only if left is negative. - */ - def % (other: Expression): Expression = mod(other) - - /** - * Indicates the range from left to right, i.e. [left, right], which can be used in columns - * selection. - * - * e.g. withColumns(1 to 3) - */ - def to (other: Expression): Expression = unresolvedCall(RANGE_TO, expr, objectToExpression(other)) - - /** - * Ternary conditional operator that decides which of two other expressions should be - * based on a evaluated boolean condition. - * - * e.g. ($"f0" > 5).?("A", "B") leads to "A" - * - * @param ifTrue expression to be evaluated if condition holds - * @param ifFalse expression to be evaluated if condition does not hold - */ - def ?(ifTrue: Expression, ifFalse: Expression): Expression = - Expressions.ifThenElse(expr, ifTrue, ifFalse).toExpr - - // scalar functions - - /** - * Removes leading and/or trailing characters from the given string. - * - * @param removeLeading if true, remove leading characters (default: true) - * @param removeTrailing if true, remove trailing characters (default: true) - * @param character string containing the character (default: " ") - * @return trimmed string - */ - def trim( - removeLeading: Boolean = true, - removeTrailing: Boolean = true, - character: Expression = valueLiteral(" ")) - : Expression = { - unresolvedCall( - TRIM, - valueLiteral(removeLeading), - valueLiteral(removeTrailing), - ApiExpressionUtils.objectToExpression(character), - expr) - } - - // Row interval type - - /** - * Creates an interval of rows. - * - * @return interval of rows - */ - def rows: Expression = toRowInterval(expr) - -} +import scala.language.implicitConversions /** * Implicit conversions from Scala literals to [[Expression]] and from [[Expression]] diff --git a/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionOperations.scala b/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionOperations.scala new file mode 100644 index 0000000000000..83b03133d5a8e --- /dev/null +++ b/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/ImplicitExpressionOperations.scala @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.api + +import org.apache.flink.annotation.PublicEvolving +import org.apache.flink.table.api.internal.BaseExpressions +import org.apache.flink.table.expressions.ApiExpressionUtils._ +import org.apache.flink.table.expressions._ +import org.apache.flink.table.functions.BuiltInFunctionDefinitions._ + +import scala.language.implicitConversions + +/** + * These are all the operations that can be used to construct an [[Expression]] AST for + * expression operations. + */ +@PublicEvolving +trait ImplicitExpressionOperations extends BaseExpressions[Expression, Expression] { + private[flink] def expr: Expression + + override def toExpr: Expression = expr + + override protected def toApiSpecificExpression(expression: Expression): Expression = expression + + /** + * Specifies a name for an expression i.e. a field. + * + * @param name name for one field + * @param extraNames additional names if the expression expands to multiple fields + * @return field with an alias + */ + def as(name: Symbol, extraNames: Symbol*): Expression = as(name.name, extraNames.map(_.name): _*) + + /** + * Boolean AND in three-valued logic. + */ + def && (other: Expression): Expression = and(other) + + /** + * Boolean OR in three-valued logic. + */ + def || (other: Expression): Expression = or(other) + + /** + * Greater than. + */ + def > (other: Expression): Expression = isGreater(other) + + /** + * Greater than or equal. + */ + def >= (other: Expression): Expression = isGreaterOrEqual(other) + + /** + * Less than. + */ + def < (other: Expression): Expression = isLess(other) + + /** + * Less than or equal. + */ + def <= (other: Expression): Expression = isLessOrEqual(other) + + /** + * Equals. + */ + def === (other: Expression): Expression = isEqual(other) + + /** + * Not equal. + */ + def !== (other: Expression): Expression = isNotEqual(other) + + /** + * Whether boolean expression is not true; returns null if boolean is null. + */ + def unary_! : Expression = unresolvedCall(NOT, expr) + + /** + * Returns negative numeric. + */ + def unary_- : Expression = Expressions.negative(expr) + + /** + * Returns numeric. + */ + def unary_+ : Expression = expr + + /** + * Returns left plus right. + */ + def + (other: Expression): Expression = plus(other) + + /** + * Returns left minus right. + */ + def - (other: Expression): Expression = minus(other) + + /** + * Returns left divided by right. + */ + def / (other: Expression): Expression = dividedBy(other) + + /** + * Returns left multiplied by right. + */ + def * (other: Expression): Expression = times(other) + + /** + * Returns the remainder (modulus) of left divided by right. + * The result is negative only if left is negative. + */ + def % (other: Expression): Expression = mod(other) + + /** + * Indicates the range from left to right, i.e. [left, right], which can be used in columns + * selection. + * + * e.g. withColumns(1 to 3) + */ + def to (other: Expression): Expression = unresolvedCall(RANGE_TO, expr, objectToExpression(other)) + + /** + * Ternary conditional operator that decides which of two other expressions should be + * based on a evaluated boolean condition. + * + * e.g. ($"f0" > 5).?("A", "B") leads to "A" + * + * @param ifTrue expression to be evaluated if condition holds + * @param ifFalse expression to be evaluated if condition does not hold + */ + def ?(ifTrue: Expression, ifFalse: Expression): Expression = + Expressions.ifThenElse(expr, ifTrue, ifFalse).toExpr + + // scalar functions + + /** + * Removes leading and/or trailing characters from the given string. + * + * @param removeLeading if true, remove leading characters (default: true) + * @param removeTrailing if true, remove trailing characters (default: true) + * @param character string containing the character (default: " ") + * @return trimmed string + */ + def trim( + removeLeading: Boolean = true, + removeTrailing: Boolean = true, + character: Expression = valueLiteral(" ")) + : Expression = { + unresolvedCall( + TRIM, + valueLiteral(removeLeading), + valueLiteral(removeTrailing), + ApiExpressionUtils.objectToExpression(character), + expr) + } + + // Row interval type + + /** + * Creates an interval of rows. + * + * @return interval of rows + */ + def rows: Expression = toRowInterval(expr) +} diff --git a/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/package.scala b/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/package.scala index 2fb4f3269eb0f..82c8cfa75edcb 100644 --- a/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/package.scala +++ b/flink-table/flink-table-api-scala/src/main/scala/org/apache/flink/table/api/package.scala @@ -18,8 +18,6 @@ package org.apache.flink.table -import org.apache.flink.table.api.{ImplicitExpressionConversions, ImplicitExpressionOperations, Table, TableEnvironment} - /** * == Table & SQL API == * @@ -39,10 +37,6 @@ import org.apache.flink.table.api.{ImplicitExpressionConversions, ImplicitExpres * * Please refer to the website documentation about how to construct and run table programs. */ -package object api /* extends ImplicitExpressionConversions */ { - - // This package object should extend from ImplicitExpressionConversions but would clash with - // "org.apache.flink.table.api.scala._" therefore we postpone splitting the package object into - // two and let users update there imports first +package object api extends ImplicitExpressionConversions { } diff --git a/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/ExpressionsConsistencyCheckTest.scala b/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/ExpressionsConsistencyCheckTest.scala index d864bdcb27ddf..c3699ad615892 100644 --- a/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/ExpressionsConsistencyCheckTest.scala +++ b/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/ExpressionsConsistencyCheckTest.scala @@ -253,8 +253,6 @@ class ExpressionsConsistencyCheckTest { def testInteroperability(): Unit = { // In most cases it should be just fine to mix the two APIs. // It should be discouraged though as it might have unforeseen side effects - object Conversions extends ImplicitExpressionConversions - import Conversions._ val expr = lit("ABC") === $"f0".plus($("f1")).trim() assertThat( diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/delegation/BlinkExecutorFactory.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/delegation/BlinkExecutorFactory.java index a193cb78430ba..56c53b439d481 100644 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/delegation/BlinkExecutorFactory.java +++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/delegation/BlinkExecutorFactory.java @@ -34,7 +34,7 @@ * Factory to create an implementation of {@link Executor} to use in a * {@link org.apache.flink.table.api.TableEnvironment}. The {@link org.apache.flink.table.api.TableEnvironment} * should use {@link #create(Map)} method that does not bind to any particular environment, - * whereas {@link org.apache.flink.table.api.scala.StreamTableEnvironment} should use + * whereas {@link org.apache.flink.table.api.bridge.scala.StreamTableEnvironment} should use * {@link #create(Map, StreamExecutionEnvironment)} as it is always backed by * some {@link StreamExecutionEnvironment} */ diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/expressions/package.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/expressions/package.scala deleted file mode 100644 index 41e0c9f5bc384..0000000000000 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/expressions/package.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.flink.table - -/** - * This package contains the base class of AST nodes and all the expression language AST classes. - * Expression trees should not be manually constructed by users. They are implicitly constructed - * from the implicit DSL conversions in - * [[org.apache.flink.table.api.scala.ImplicitExpressionConversions]] and - * [[org.apache.flink.table.api.scala.ImplicitExpressionOperations]]. For the Java API, - * expression trees should be generated from a string parser that parses expressions and creates - * AST nodes. - */ -package object expressions diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowEmitStrategy.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowEmitStrategy.scala index 07b30f488b860..d59b86f693042 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowEmitStrategy.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowEmitStrategy.scala @@ -24,24 +24,24 @@ import org.apache.flink.table.api.{TableConfig, TableException} import org.apache.flink.table.planner.plan.logical.{LogicalWindow, SessionGroupWindow} import org.apache.flink.table.planner.plan.utils.AggregateUtil.isRowtimeAttribute import org.apache.flink.table.planner.utils.TableConfigUtils.getMillisecondFromConfigDuration +import org.apache.flink.table.planner.{JBoolean, JLong} import org.apache.flink.table.runtime.operators.window.TimeWindow import org.apache.flink.table.runtime.operators.window.triggers._ -import java.lang.{Boolean, Long} import java.time.Duration class WindowEmitStrategy( - isEventTime: Boolean, - isSessionWindow: Boolean, - earlyFireDelay: Long, - earlyFireDelayEnabled: Boolean, - lateFireDelay: Long, - lateFireDelayEnabled: Boolean, - allowLateness: Long) { + isEventTime: JBoolean, + isSessionWindow: JBoolean, + earlyFireDelay: JLong, + earlyFireDelayEnabled: JBoolean, + lateFireDelay: JLong, + lateFireDelayEnabled: JBoolean, + allowLateness: JLong) { checkValidation() - def getAllowLateness: Long = allowLateness + def getAllowLateness: JLong = allowLateness private def checkValidation(): Unit = { if (isSessionWindow && (earlyFireDelayEnabled || lateFireDelayEnabled)) { @@ -61,7 +61,7 @@ class WindowEmitStrategy( } } - def produceUpdates: Boolean = { + def produceUpdates: JBoolean = { if (isEventTime) { earlyFireDelayEnabled || lateFireDelayEnabled } else { @@ -110,8 +110,8 @@ class WindowEmitStrategy( } private def createTriggerFromInterval( - enableDelayEmit: Boolean, - interval: Long): Option[Trigger[TimeWindow]] = { + enableDelayEmit: JBoolean, + interval: JLong): Option[Trigger[TimeWindow]] = { if (!enableDelayEmit) { None } else { @@ -123,7 +123,7 @@ class WindowEmitStrategy( } } - private def intervalToString(enableDelayEmit: Boolean, interval: Long): String = { + private def intervalToString(enableDelayEmit: JBoolean, interval: JLong): String = { if (!enableDelayEmit) { null } else { @@ -171,9 +171,9 @@ object WindowEmitStrategy { // It is a experimental config, will may be removed later. @Experimental - val TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED: ConfigOption[Boolean] = + val TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED: ConfigOption[JBoolean] = key("table.exec.emit.early-fire.enabled") - .defaultValue(Boolean.valueOf(false)) + .defaultValue(Boolean.box(false)) .withDescription("Specifies whether to enable early-fire emit." + "Early-fire is an emit strategy before watermark advanced to end of window.") @@ -190,9 +190,9 @@ object WindowEmitStrategy { // It is a experimental config, will may be removed later. @Experimental - val TABLE_EXEC_EMIT_LATE_FIRE_ENABLED: ConfigOption[Boolean] = + val TABLE_EXEC_EMIT_LATE_FIRE_ENABLED: ConfigOption[JBoolean] = key("table.exec.emit.late-fire.enabled") - .defaultValue(Boolean.valueOf(false)) + .defaultValue(Boolean.box(false)) .withDescription("Specifies whether to enable late-fire emit. " + "Late-fire is an emit strategy after watermark advanced to end of window.") diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/api/EnvironmentTest.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/api/EnvironmentTest.java index 80801ca3e8719..d83d4fe49f4d6 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/api/EnvironmentTest.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/api/EnvironmentTest.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.junit.Test; diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/catalog/CatalogITCase.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/catalog/CatalogITCase.java index e85720da38a9f..4e8e95b06741d 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/catalog/CatalogITCase.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/catalog/CatalogITCase.java @@ -21,7 +21,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.GenericInMemoryCatalog; import org.junit.Test; diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala index a90b2c5b83732..bc6b46344f484 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala @@ -23,8 +23,8 @@ import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment => ScalaStreamExecutionEnvironment} import org.apache.flink.table.api.internal.{TableEnvironmentImpl, TableEnvironmentInternal} -import org.apache.flink.table.api.java.StreamTableEnvironment -import org.apache.flink.table.api.scala.{StreamTableEnvironment => ScalaStreamTableEnvironment, _} +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.{StreamTableEnvironment => ScalaStreamTableEnvironment, _} import org.apache.flink.table.planner.factories.utils.TestCollectionTableFactory import org.apache.flink.table.planner.runtime.utils.{TableEnvUtil, TestingAppendSink} import org.apache.flink.table.planner.utils.TableTestUtil.{readFromResource, replaceStageId} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala index 2877de87acb50..e6ec3bee80d65 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.Types.STRING import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.environment.LocalStreamEnvironment import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala.{StreamTableEnvironment, _} +import org.apache.flink.table.api.bridge.scala.{StreamTableEnvironment, _} import org.apache.flink.table.catalog.{GenericInMemoryCatalog, ObjectPath} import org.apache.flink.table.planner.operations.SqlConversionException import org.apache.flink.table.planner.runtime.stream.sql.FunctionITCase.TestUDF diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableITCase.scala index 865a7bbc56bc3..01f400d3743d9 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/TableITCase.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.api import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.java.StreamTableEnvironment +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment import org.apache.flink.table.planner.utils.TestTableSourceSinks import org.apache.flink.types.Row import org.apache.flink.util.TestLogger diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala index b60799d1d9722..e68629aa51a2b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala @@ -19,12 +19,12 @@ package org.apache.flink.table.api.batch import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.types.logical.{BigIntType, IntType, VarCharType} + import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala index ee7222d0c85bf..f822f2b5d8b3d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala @@ -19,17 +19,17 @@ package org.apache.flink.table.api.stream import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.types.logical.{BigIntType, IntType, VarCharType} + import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.{Before, Test} -import java.sql.Timestamp -import org.apache.flink.table.api.internal.TableEnvironmentInternal +import java.sql.Timestamp @RunWith(classOf[Parameterized]) class ExplainTest(extended: Boolean) extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala index 5fa8366d0ebaa..3fcab9f36ffd5 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala @@ -20,11 +20,11 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvg -import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils.ToMillis import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction +import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils.ToMillis import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala index e6af002aeffa9..fc68e5a813946 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.OverAgg0 import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala index 953f0f311565b..7a3a2602dbf56 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala @@ -18,11 +18,11 @@ package org.apache.flink.table.api.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func0 import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.OverAgg0 import org.apache.flink.table.planner.utils.TableTestBase + import org.junit.Test class UserDefinedFunctionValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/codegen/agg/AggTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/codegen/agg/AggTestBase.scala index d2464a714f0e4..19716cce909b6 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/codegen/agg/AggTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/codegen/agg/AggTestBase.scala @@ -22,7 +22,7 @@ import org.apache.flink.api.common.functions.RuntimeContext import org.apache.flink.streaming.api.environment.LocalStreamEnvironment import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment => ScalaStreamExecEnv} import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment import org.apache.flink.table.api.{DataTypes, EnvironmentSettings} import org.apache.flink.table.planner.calcite.{FlinkTypeFactory, FlinkTypeSystem} import org.apache.flink.table.planner.codegen.CodeGeneratorContext diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ArrayTypeTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ArrayTypeTest.scala index db59a8f8323ae..cfc798d2b5ba5 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ArrayTypeTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ArrayTypeTest.scala @@ -18,15 +18,14 @@ package org.apache.flink.table.planner.expressions -import org.apache.flink.table.api.DataTypes -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.ArrayTypeTestBase import org.apache.flink.table.planner.utils.DateTimeTestUtil.{localDate, localDateTime, localTime => gLocalTime} -import java.time.{LocalDateTime => JLocalDateTime} - import org.junit.Test +import java.time.{LocalDateTime => JLocalDateTime} + class ArrayTypeTest extends ArrayTypeTestBase { @Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/CompositeAccessTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/CompositeAccessTest.scala index 43b8160a84b73..4a2e7f0ea1d90 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/CompositeAccessTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/CompositeAccessTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.expressions -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.CompositeTypeTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala index 2a888981b8966..be8220eb5d2be 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.expressions import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, Types} +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.ApiExpressionUtils.valueLiteral import org.apache.flink.table.planner.expressions.utils.ExpressionTestBase import org.apache.flink.table.runtime.types.TypeInfoLogicalTypeConverter.fromLogicalTypeToTypeInfo diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/LiteralTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/LiteralTest.scala index 5158b25803a52..7100f6158777a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/LiteralTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/LiteralTest.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.planner.expressions import org.apache.flink.api.common.typeinfo.{TypeInformation, Types} import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.expressions.utils.{ExpressionTestBase, Func3} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/MapTypeTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/MapTypeTest.scala index 29934320a8a86..94e1b4dac189e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/MapTypeTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/MapTypeTest.scala @@ -18,16 +18,15 @@ package org.apache.flink.table.planner.expressions -import org.apache.flink.table.api.DataTypes -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{DataTypes, _} import org.apache.flink.table.expressions.ApiExpressionUtils.valueLiteral import org.apache.flink.table.planner.expressions.utils.MapTypeTestBase import org.apache.flink.table.planner.utils.DateTimeTestUtil.{localDate, localDateTime, localTime => gLocalTime} -import java.time.{LocalDateTime => JLocalTimestamp} - import org.junit.Test +import java.time.{LocalDateTime => JLocalTimestamp} + class MapTypeTest extends MapTypeTestBase { @Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala index 9cd8c63229221..890b1b7c57b12 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/NonDeterministicTests.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.expressions import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.ExpressionTestBase import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/RowTypeTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/RowTypeTest.scala index 970d7d6876c8a..89353949829f8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/RowTypeTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/RowTypeTest.scala @@ -18,11 +18,11 @@ package org.apache.flink.table.planner.expressions -import org.apache.flink.table.api.{DataTypes, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.codegen.CodeGenException import org.apache.flink.table.planner.expressions.utils.RowTypeTestBase import org.apache.flink.table.planner.utils.DateTimeTestUtil.{localDate, localDateTime, localTime => gLocalTime} + import org.junit.Test class RowTypeTest extends RowTypeTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala index 9c84b8f44e390..c0615342c894b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala @@ -18,10 +18,10 @@ package org.apache.flink.table.planner.expressions -import org.apache.flink.table.api.scala.{currentDate, currentTime, currentTimestamp, localTime, localTimestamp, nullOf, temporalOverlaps, _} -import org.apache.flink.table.api.{DataTypes, Types} +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.{Expression, ExpressionParser, TimeIntervalUnit, TimePointUnit} import org.apache.flink.table.planner.expressions.utils.ScalarTypesTestBase + import org.junit.Test class ScalarFunctionsTest extends ScalarTypesTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala index ce4e73342ee10..8040704ce4930 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala @@ -20,12 +20,12 @@ package org.apache.flink.table.planner.expressions import org.apache.flink.api.common.typeinfo.Types import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.DataTypes -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.TimeIntervalUnit import org.apache.flink.table.planner.expressions.utils.ExpressionTestBase import org.apache.flink.table.planner.utils.DateTimeTestUtil import org.apache.flink.table.planner.utils.DateTimeTestUtil._ +import org.apache.flink.table.runtime.typeutils.{LegacyInstantTypeInfo, LegacyLocalDateTimeTypeInfo} import org.apache.flink.table.typeutils.TimeIntervalTypeInfo import org.apache.flink.types.Row @@ -35,7 +35,6 @@ import java.sql.Timestamp import java.text.SimpleDateFormat import java.time.{Instant, ZoneId, ZoneOffset} import java.util.{Locale, TimeZone} -import org.apache.flink.table.runtime.typeutils.{LegacyInstantTypeInfo, LegacyLocalDateTimeTypeInfo} class TemporalTypesTest extends ExpressionTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/UserDefinedScalarFunctionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/UserDefinedScalarFunctionTest.scala index 2e9e6eab4e9a0..409bdf3cbbab1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/UserDefinedScalarFunctionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/UserDefinedScalarFunctionTest.scala @@ -20,12 +20,11 @@ package org.apache.flink.table.planner.expressions import org.apache.flink.api.common.typeinfo.{BasicArrayTypeInfo, BasicTypeInfo, TypeInformation} import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, Types, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.expressions.utils.{ExpressionTestBase, _} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions._ -import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils.{DateFunction, DateTimeFunction, LocalDateFunction, LocalTimeFunction, TimeFunction, TimestampFunction} +import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils._ import org.apache.flink.table.planner.utils.DateTimeTestUtil import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala index 6e1da9cf0e102..8469857c1d1f7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala @@ -25,7 +25,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.java.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl import org.apache.flink.table.api.{EnvironmentSettings, TableConfig} import org.apache.flink.table.data.RowData import org.apache.flink.table.data.binary.BinaryRowData diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ArrayTypeValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ArrayTypeValidationTest.scala index 2afbf6fe3eeec..5b098ceee5528 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ArrayTypeValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ArrayTypeValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.ArrayTypeTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/CompositeAccessValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/CompositeAccessValidationTest.scala index 687700e466f06..96f3efb1c4fbf 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/CompositeAccessValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/CompositeAccessValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.CompositeTypeTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/MapTypeValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/MapTypeValidationTest.scala index 52a62c7a1503d..a01e32162ed6d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/MapTypeValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/MapTypeValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.MapTypeTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/RowTypeValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/RowTypeValidationTest.scala index f72de54ad1f5c..9e93e13a85624 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/RowTypeValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/RowTypeValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.expressions.validation -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{SqlParserException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.RowTypeTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarFunctionsValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarFunctionsValidationTest.scala index f38ab9362ac3f..2634b686c6654 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarFunctionsValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarFunctionsValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.expressions.validation -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{SqlParserException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.TimePointUnit import org.apache.flink.table.planner.codegen.CodeGenException import org.apache.flink.table.planner.expressions.utils.ScalarTypesTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarOperatorsValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarOperatorsValidationTest.scala index 9bc2ea87228e1..f3b5fffa16d8b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarOperatorsValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/validation/ScalarOperatorsValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.ScalarOperatorsTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/match/PatternTranslatorTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/match/PatternTranslatorTestBase.scala index c12361e22d62c..f8df461ac90df 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/match/PatternTranslatorTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/match/PatternTranslatorTestBase.scala @@ -23,9 +23,9 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.cep.pattern.Pattern import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream} import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.TableConfig +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala.{StreamTableEnvironment, _} import org.apache.flink.table.data.RowData import org.apache.flink.table.expressions.Expression import org.apache.flink.table.planner.calcite.FlinkPlannerImpl @@ -35,6 +35,7 @@ import org.apache.flink.table.planner.utils.TableTestUtil import org.apache.flink.table.types.logical.{IntType, RowType} import org.apache.flink.types.Row import org.apache.flink.util.TestLogger + import org.apache.calcite.rel.RelNode import org.apache.calcite.tools.RelBuilder import org.junit.Assert._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/CalcTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/CalcTest.scala index 431dc6b8cfa29..8d3130fe87afa 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/CalcTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/CalcTest.scala @@ -21,8 +21,7 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.MyPojo import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DagOptimizationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DagOptimizationTest.scala index a25300824795c..3aaf80cf0440f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DagOptimizationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DagOptimizationTest.scala @@ -18,15 +18,16 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.plan.optimize.RelNodeBlockPlanBuilder import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.NonDeterministicUdf import org.apache.flink.table.planner.utils.{TableFunc1, TableTestBase} import org.apache.flink.table.types.logical._ + import org.junit.Test -import java.sql.Timestamp -import org.apache.flink.table.api.internal.TableEnvironmentInternal +import java.sql.Timestamp class DagOptimizationTest extends TableTestBase { private val util = batchTestUtil() diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.scala index 2b31551200f61..c538dc1299039 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LegacySinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LegacySinkTest.scala index d50b0ebe25e9d..2719992ddf20a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LegacySinkTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LegacySinkTest.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.optimize.RelNodeBlockPlanBuilder import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.types.logical.{BigIntType, IntType} + import org.junit.Test class LegacySinkTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LimitTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LimitTest.scala index 95ec361b50862..788eb3a93ab25 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LimitTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/LimitTest.scala @@ -21,9 +21,9 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{INT_TYPE_INFO, LONG_TYPE_INFO, STRING_TYPE_INFO} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{SqlParserException, TableSchema} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{SqlParserException, TableSchema, _} import org.apache.flink.table.planner.utils.{TableTestBase, TestLimitableTableSource} + import org.junit.Test class LimitTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/PartitionableSinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/PartitionableSinkTest.scala index ed1c336bcad87..cfcf38bee1ad4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/PartitionableSinkTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/PartitionableSinkTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/RankTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/RankTest.scala index 22da0a1a54293..62fb3f691b59c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/RankTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/RankTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SetOperatorsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SetOperatorsTest.scala index a0a268cb9b336..625f83e3d052f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SetOperatorsTest.scala @@ -21,9 +21,8 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.utils.NonPojo import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortLimitTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortLimitTest.scala index de2692d6d2f7e..cebda191b2f55 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortLimitTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortLimitTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.rules.physical.batch.BatchExecSortRule.TABLE_EXEC_SORT_RANGE_ENABLED import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortTest.scala index 10a19a84de526..cb40364f2b5f0 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SortTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.rules.physical.batch.BatchExecSortRule.TABLE_EXEC_SORT_RANGE_ENABLED import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SubplanReuseTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SubplanReuseTest.scala index d20f26e5bdd06..8e7b767379ab8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SubplanReuseTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/SubplanReuseTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.functions.aggfunctions.FirstValueAggFunction.IntFirstValueAggFunction import org.apache.flink.table.planner.functions.aggfunctions.LastValueAggFunction.LongLastValueAggFunction import org.apache.flink.table.planner.plan.rules.physical.batch.BatchExecSortMergeJoinRule diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableScanTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableScanTest.scala index a8cdd7c279872..6a92a5aea3be3 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableScanTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableScanTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{DataTypes, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.descriptors.{FileSystem, OldCsv, Schema} import org.apache.flink.table.planner.expressions.utils.Func0 import org.apache.flink.table.planner.utils.TableTestBase + import org.junit.{Before, Test} class TableScanTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala index df11fd97ec302..fc544daca9638 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.RelNodeBlockPlanBuilder import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.types.logical.{BigIntType, IntType} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/UnionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/UnionTest.scala index dccd19eff771a..bbff45e288370 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/UnionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/UnionTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateTestBase.scala index ace02cb1eb2ba..0f35b8ccc6703 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateTestBase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql.agg import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{TableException, Types} import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{VarSum1AggFunction, VarSum2AggFunction} import org.apache.flink.table.planner.utils.{BatchTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.scala index c981706b927da..6d9a164aa148a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/GroupingSetsTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql.agg import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/OverAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/OverAggregateTest.scala index b140072553782..6e106852d54b4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/OverAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/OverAggregateTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.sql.agg import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.OverAgg0 import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/WindowAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/WindowAggregateTest.scala index b27164b4649ee..e933100aa6702 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/WindowAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/agg/WindowAggregateTest.scala @@ -18,9 +18,8 @@ package org.apache.flink.table.planner.plan.batch.sql.agg import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.planner.utils.{AggregatePhaseStrategy, CountAggFunction, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/JoinTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/JoinTestBase.scala index 855ccfc2204f6..67350d4d631e4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/JoinTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/JoinTestBase.scala @@ -18,7 +18,8 @@ package org.apache.flink.table.planner.plan.batch.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.planner.utils.{BatchTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/LookupJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/LookupJoinTest.scala index 7e5e84687a42b..140a4b288db6a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/LookupJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/LookupJoinTest.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.planner.plan.batch.sql.join import org.apache.flink.api.scala._ import org.apache.flink.table.api._ import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.optimize.program.FlinkBatchProgram import org.apache.flink.table.planner.plan.stream.sql.join.TestTemporalTable import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTestBase.scala index 7e7732dd939f8..6dc596ec81460 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTestBase.scala @@ -18,7 +18,8 @@ package org.apache.flink.table.planner.plan.batch.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.StringSplit import org.apache.flink.table.planner.utils.{BatchTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SingleRowJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SingleRowJoinTest.scala index 57a44a357ad7b..64217205280ab 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SingleRowJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/SingleRowJoinTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/TemporalJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/TemporalJoinTest.scala index e1ae532d72227..b50b5120ab7f7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/TemporalJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/join/TemporalJoinTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.batch.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{BatchTableTestUtil, TableTestBase} import org.hamcrest.Matchers.containsString diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/AggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/AggregateTest.scala index 9ec38ff647c04..416ef75756387 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/AggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/AggregateTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CalcTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CalcTest.scala index 2dff2b1b75a70..4770969132aa9 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CalcTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CalcTest.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala.createTypeInformation -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.plan.batch.table.CalcTest.{MyHashCode, TestCaseClass, WC, giveMeCaseClass} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/ColumnFunctionsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/ColumnFunctionsTest.scala index 2c16ebc41fc05..0984e47070076 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/ColumnFunctionsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/ColumnFunctionsTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CorrelateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CorrelateTest.scala index 7edf3d3070b51..049921084392d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CorrelateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/CorrelateTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.FlinkBatchProgram import org.apache.flink.table.planner.utils.{MockPythonTableFunction, TableFunc0, TableFunc1, TableTestBase} + import org.apache.calcite.rel.rules.{CalcMergeRule, FilterCalcMergeRule, ProjectCalcMergeRule} import org.apache.calcite.tools.RuleSets import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/GroupWindowTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/GroupWindowTest.scala index fc951b41022ab..9a95d3ea7f079 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/GroupWindowTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/GroupWindowTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Slide, TableException, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/JoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/JoinTest.scala index 46555bdd8a6e5..abbe6eacaad2c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/JoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/JoinTest.scala @@ -19,13 +19,12 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.plan.batch.table.JoinTest.Merger import org.apache.flink.table.planner.utils.TableTestBase -import org.junit.{Ignore, Test} +import org.junit.Test class JoinTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/PythonCalcTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/PythonCalcTest.scala index 4c01bfd967696..b286223f9eb68 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/PythonCalcTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/PythonCalcTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.TableTestBase + import org.junit.{Before, Test} class PythonCalcTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/SetOperatorsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/SetOperatorsTest.scala index 50206d3de7539..284b82a365702 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/SetOperatorsTest.scala @@ -21,7 +21,7 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.NonPojo import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/TemporalTableJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/TemporalTableJoinTest.scala index d6646ad4cbb48..030e124edc698 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/TemporalTableJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/TemporalTableJoinTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} import org.hamcrest.Matchers.containsString diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/AggregateStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/AggregateStringExpressionTest.scala index b88544f8b19bd..e31c5fb1d0cd1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/AggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/AggregateStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset import org.apache.flink.table.planner.utils.{CountAggFunction, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CalcStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CalcStringExpressionTest.scala index a95c08a07835e..e30b829e295d1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CalcStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CalcStringExpressionTest.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types import org.apache.flink.table.api.Types._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.CollectionBatchExecTable.CustomType import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CorrelateStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CorrelateStringExpressionTest.scala index aea7ac00afaa5..262582235095d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CorrelateStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/CorrelateStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils.{HierarchyTableFunction, PojoTableFunc, TableFunc1, TableFunc2, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/JoinStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/JoinStringExpressionTest.scala index 0649c833b3a00..d31ed7b7511e4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/JoinStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/JoinStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SetOperatorsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SetOperatorsTest.scala index 7df5d19a73c51..3c21c5ca5b481 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SetOperatorsTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SortStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SortStringExpressionTest.scala index 0cfb75896115b..566c8ded89763 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SortStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/stringexpr/SortStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/AggregateValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/AggregateValidationTest.scala index de239c97ce87f..0302359ce35e7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/AggregateValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/AggregateValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CalcValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CalcValidationTest.scala index e6803e699bcbd..b07fbcee7210f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CalcValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CalcValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Assert._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CorrelateValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CorrelateValidationTest.scala index 7105af7413146..3b23317513f86 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CorrelateValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/CorrelateValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{TableFunc1, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/GroupWindowValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/GroupWindowValidationTest.scala index 0b1112764d7f2..7760bb3fbc18f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/GroupWindowValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/GroupWindowValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble, ValidationException} +import org.apache.flink.table.api.{Session, Slide, Tumble, ValidationException, _} import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/JoinValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/JoinValidationTest.scala index df79bceb6bdb6..06ade02ca18f2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/JoinValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/JoinValidationTest.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, TableException, ValidationException} import org.apache.flink.table.planner.runtime.utils.CollectionBatchExecTable import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/OverWindowValidationTest.scala index 049d28c11b6e4..a2a67418d6617 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/OverWindowValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Tumble, ValidationException} +import org.apache.flink.table.api.{Tumble, ValidationException, _} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.OverAgg0 import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SetOperatorsValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SetOperatorsValidationTest.scala index 43c98089ab3f5..38c257792a232 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SetOperatorsValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SetOperatorsValidationTest.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, ValidationException} import org.apache.flink.table.planner.runtime.utils.CollectionBatchExecTable import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SortValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SortValidationTest.scala index d5cc4b973554d..713a4e1c764dd 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SortValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/table/validation/SortValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/DistinctAggregateTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/DistinctAggregateTestBase.scala index d631892f60120..6ec992fb5394c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/DistinctAggregateTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/DistinctAggregateTestBase.scala @@ -20,9 +20,9 @@ package org.apache.flink.table.planner.plan.common import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{BatchTableTestUtil, TableTestBase} + import org.junit.{Before, Test} abstract class DistinctAggregateTestBase extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/UnnestTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/UnnestTestBase.scala index 28a5e509489b7..a695b325292b2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/UnnestTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/UnnestTestBase.scala @@ -20,8 +20,7 @@ package org.apache.flink.table.planner.plan.common import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/ViewsExpandingTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/ViewsExpandingTest.scala index 074a072f79d84..5bdb6b5ad6d25 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/ViewsExpandingTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/common/ViewsExpandingTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.common import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, TableSchema} +import org.apache.flink.table.api._ import org.apache.flink.table.catalog.{CatalogView, CatalogViewImpl, ObjectPath} import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil, TableTestUtilBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcPythonCorrelateTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcPythonCorrelateTransposeRuleTest.scala index 5f97cfaca43b7..b0563c5ddbd35 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcPythonCorrelateTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcPythonCorrelateTransposeRuleTest.scala @@ -18,14 +18,15 @@ package org.apache.flink.table.planner.plan.rules.logical -import org.apache.calcite.plan.hep.HepMatchOrder import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.rules.FlinkStreamRuleSets import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.{MockPythonTableFunction, TableTestBase} + +import org.apache.calcite.plan.hep.HepMatchOrder import org.junit.{Before, Test} class CalcPythonCorrelateTransposeRuleTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcRankTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcRankTransposeRuleTest.scala index b640272aa0a2f..760e7c42c9f82 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcRankTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/CalcRankTransposeRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.FlinkStreamProgram import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ConvertToNotInOrInRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ConvertToNotInOrInRuleTest.scala index ffe103eea91de..37c29a1958a58 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ConvertToNotInOrInRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ConvertToNotInOrInRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.scala index ff964ab3a9919..030ed15f4983f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/DecomposeGroupingSetsRuleTest.scala @@ -20,8 +20,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.FlinkBatchProgram import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala index 13a42775b9f7d..58c92a3c5c8d8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.functions.python.{PythonEnv, PythonFunction} import org.apache.flink.table.planner.expressions.utils.{Func1, RichFunc1} import org.apache.flink.table.planner.utils.TableTestBase + import org.junit.Test /** diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala index cbf9077189979..d2f205a9dd6d5 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala @@ -20,9 +20,8 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkGroupProgramBuilder, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} +import org.apache.flink.table.api._ +import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.scala index 83efa64ba052b..6ff25ff49e539 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.scala @@ -19,10 +19,9 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions -import org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalAggregate, FlinkLogicalCalc, FlinkLogicalExpand, FlinkLogicalJoin, FlinkLogicalLegacySink, FlinkLogicalLegacyTableSourceScan, FlinkLogicalValues} +import org.apache.flink.table.planner.plan.nodes.logical._ import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.rules.FlinkBatchRuleSets import org.apache.flink.table.planner.plan.stats.FlinkStatistic @@ -30,7 +29,7 @@ import org.apache.flink.table.planner.utils.TableTestBase import com.google.common.collect.ImmutableSet import org.apache.calcite.plan.hep.HepMatchOrder -import org.apache.calcite.rel.rules.{FilterCalcMergeRule, FilterToCalcRule, ProjectCalcMergeRule, ProjectToCalcRule, ReduceExpressionsRule} +import org.apache.calcite.rel.rules._ import org.apache.calcite.tools.RuleSets import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkCalcMergeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkCalcMergeRuleTest.scala index c2f26ede492bb..54fd6b260a837 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkCalcMergeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkCalcMergeRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions import org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalCalc, FlinkLogicalLegacyTableSourceScan} import org.apache.flink.table.planner.plan.optimize.program._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala index 8b229100d179d..15b7ae3936231 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRuleTest.scala index cfeea9a3c0b3b..991d0f4b8039f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala index dca814ce33c6a..48d4bbcf4132c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLimit0RemoveRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLimit0RemoveRuleTest.scala index de371048e9951..1582d28685974 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLimit0RemoveRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLimit0RemoveRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForConstantRangeTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForConstantRangeTest.scala index 199aec58f2e43..8cdbf245fbb7b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForConstantRangeTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForConstantRangeTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.FlinkBatchProgram import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForRangeEndTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForRangeEndTest.scala index d69d44cc9d986..0eeeede55e247 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForRangeEndTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkLogicalRankRuleForRangeEndTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.FlinkStreamProgram import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkPruneEmptyRulesTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkPruneEmptyRulesTest.scala index da7b6f53d1f13..96acaf780aab6 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkPruneEmptyRulesTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkPruneEmptyRulesTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinFilterTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinFilterTransposeRuleTest.scala index 6668d71c0f724..154bd803a821b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinFilterTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinFilterTransposeRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinJoinTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinJoinTransposeRuleTest.scala index bc59086d3625a..709cfaea9aff4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinJoinTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinJoinTransposeRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinProjectTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinProjectTransposeRuleTest.scala index 7cc204c43107f..febcdd4a7db1b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinProjectTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkSemiAntiJoinProjectTransposeRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionEqualityTransferRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionEqualityTransferRuleTest.scala index ec5ca57fceaca..184acd6f172a2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionEqualityTransferRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionEqualityTransferRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} @@ -27,7 +27,6 @@ import org.apache.calcite.plan.hep.HepMatchOrder import org.apache.calcite.tools.RuleSets import org.junit.{Before, Test} - /** * Test for [[JoinConditionEqualityTransferRule]]. */ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionTypeCoerceRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionTypeCoerceRuleTest.scala index 3b22b80989b8c..ac779a8df7385 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionTypeCoerceRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinConditionTypeCoerceRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala index 0da08f036611f..51b61b5bc668f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ProjectSemiAntiJoinTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ProjectSemiAntiJoinTransposeRuleTest.scala index 64f2e27022d0c..a2e7f5508b6a0 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ProjectSemiAntiJoinTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ProjectSemiAntiJoinTransposeRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.StringSplit import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PruneAggregateCallRuleTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PruneAggregateCallRuleTestBase.scala index d60b77453970e..4382ff0d0321e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PruneAggregateCallRuleTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PruneAggregateCallRuleTestBase.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.utils.{BatchTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRuleTest.scala index 260c9b91b1bd7..895c47c1d5b9c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCalcSplitRuleTest.scala @@ -18,14 +18,15 @@ package org.apache.flink.table.planner.plan.rules.logical -import org.apache.calcite.plan.hep.HepMatchOrder import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.rules.{FlinkBatchRuleSets, FlinkStreamRuleSets} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.{BooleanPandasScalarFunction, BooleanPythonScalarFunction, PandasScalarFunction, PythonScalarFunction} import org.apache.flink.table.planner.utils.TableTestBase + +import org.apache.calcite.plan.hep.HepMatchOrder import org.junit.{Before, Test} /** diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCorrelateSplitRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCorrelateSplitRuleTest.scala index 92d4f3ef45940..1ccc3f8e503b2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCorrelateSplitRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/PythonCorrelateSplitRuleTest.scala @@ -18,14 +18,15 @@ package org.apache.flink.table.planner.plan.rules.logical -import org.apache.calcite.plan.hep.HepMatchOrder import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.rules.FlinkStreamRuleSets import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.{MockPythonTableFunction, TableFunc1, TableTestBase} + +import org.apache.calcite.plan.hep.HepMatchOrder import org.junit.{Before, Test} class PythonCorrelateSplitRuleTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RankNumberColumnRemoveRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RankNumberColumnRemoveRuleTest.scala index 304c2eb3a0027..151bfb1e66503 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RankNumberColumnRemoveRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RankNumberColumnRemoveRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.FlinkStreamProgram import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceIntersectWithSemiJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceIntersectWithSemiJoinRuleTest.scala index 7e84a0f247005..3bfe5451a80be 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceIntersectWithSemiJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceIntersectWithSemiJoinRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceMinusWithAntiJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceMinusWithAntiJoinRuleTest.scala index de46cab416c45..cc4bcbe624608 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceMinusWithAntiJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ReplaceMinusWithAntiJoinRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteCoalesceRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteCoalesceRuleTest.scala index 6391d8b74a1d1..4e311d2957e3b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteCoalesceRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteCoalesceRuleTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.codegen.CodeGenException import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.plan.rules.FlinkBatchRuleSets diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteIntersectAllRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteIntersectAllRuleTest.scala index 1a2709cd88a1c..af08ad963e770 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteIntersectAllRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteIntersectAllRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMinusAllRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMinusAllRuleTest.scala index da7fc3e66591a..2382f6b2b51fa 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMinusAllRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMinusAllRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala index 3e0e6380b6d6b..a69e9aa9c731e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala @@ -18,8 +18,8 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkGroupProgramBuilder, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} +import org.apache.flink.table.api._ +import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.utils.TableTestBase import org.apache.calcite.plan.hep.HepMatchOrder diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.scala index cda42eeae1a51..100cdd3e6d8d4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyFilterConditionRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContext, FlinkChainedProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.scala index 7f60a4e080779..9e3f1f2489d63 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SimplifyJoinConditionRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, FlinkHepRuleSetProgramBuilder, HEP_RULES_EXECUTION_TYPE} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRuleTest.scala index c5a9658856444..48311671ecf00 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRuleTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.optimize.program.FlinkStreamProgram import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRuleTest.scala index 6147c45cd949f..51dd3bf9f0d53 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromCorrelateRuleTest.scala @@ -18,14 +18,15 @@ package org.apache.flink.table.planner.plan.rules.logical -import org.apache.calcite.plan.hep.HepMatchOrder import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.rules.FlinkBatchRuleSets import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.{TableFunc2, TableTestBase} + +import org.apache.calcite.plan.hep.HepMatchOrder import org.junit.{Before, Test} /** diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromJoinRuleTest.scala index cf8f7a2c6598c..7cd4f732355a1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/SplitPythonConditionFromJoinRuleTest.scala @@ -18,14 +18,15 @@ package org.apache.flink.table.planner.plan.rules.logical -import org.apache.calcite.plan.hep.HepMatchOrder import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.nodes.FlinkConventions import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.plan.rules.FlinkBatchRuleSets import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.TableTestBase + +import org.apache.calcite.plan.hep.HepMatchOrder import org.junit.{Before, Test} /** diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/WindowGroupReorderRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/WindowGroupReorderRuleTest.scala index 43f04cb6088b4..375b6f4072ed4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/WindowGroupReorderRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/WindowGroupReorderRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.optimize.program._ import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/FlinkRewriteSubQueryRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/FlinkRewriteSubQueryRuleTest.scala index 3c8e328b8a02f..e52219776e822 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/FlinkRewriteSubQueryRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/FlinkRewriteSubQueryRuleTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical.subquery import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.scala index b2643179f29b4..6be05193336a3 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical.subquery import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.StringSplit import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.scala index cf9e0e56a33f5..780e1ca1db7e5 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical.subquery import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.StringSplit import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.scala index 86fb4f342bb17..30116cfd0f37f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/subquery/SubqueryCorrelateVariablesValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical.subquery import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/EnforceLocalAggRuleTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/EnforceLocalAggRuleTestBase.scala index e5bde416d16b7..4acd8ba51b30c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/EnforceLocalAggRuleTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/EnforceLocalAggRuleTestBase.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.rules.physical.batch import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalHashAggRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalHashAggRuleTest.scala index d155b5cb6246c..6abf87ca2d3c2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalHashAggRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalHashAggRuleTest.scala @@ -18,8 +18,8 @@ package org.apache.flink.table.planner.plan.rules.physical.batch import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalRankRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalRankRuleTest.scala index d8ddcc3dd6249..3571792cf6dec 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalRankRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalRankRuleTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.rules.physical.batch import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalSortAggRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalSortAggRuleTest.scala index 2957254014352..69756d5985d11 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalSortAggRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/batch/RemoveRedundantLocalSortAggRuleTest.scala @@ -18,8 +18,8 @@ package org.apache.flink.table.planner.plan.rules.physical.batch import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/stream/ChangelogModeInferenceTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/stream/ChangelogModeInferenceTest.scala index 07aac629656ac..0e66a5ea382a8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/stream/ChangelogModeInferenceTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/physical/stream/ChangelogModeInferenceTest.scala @@ -20,9 +20,8 @@ package org.apache.flink.table.planner.plan.rules.physical.stream import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail +import org.apache.flink.table.api.{ExplainDetail, _} import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithAggTestBase.{AggMode, LocalGlobalOff, LocalGlobalOn} import org.apache.flink.table.planner.utils.{AggregatePhaseStrategy, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/CalcTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/CalcTest.scala index 3c7f408228cc6..14380cf7a9a76 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/CalcTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/CalcTest.scala @@ -20,8 +20,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.MyPojo import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DagOptimizationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DagOptimizationTest.scala index 45ab089a412dc..8bfd4fd4ddd06 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DagOptimizationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DagOptimizationTest.scala @@ -18,13 +18,13 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.optimize.RelNodeBlockPlanBuilder import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.NonDeterministicUdf import org.apache.flink.table.planner.utils.{TableFunc1, TableTestBase} import org.apache.flink.table.types.logical.{BigIntType, IntType, VarCharType} + import org.junit.Test class DagOptimizationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTest.scala index 8f95c4d6a1c67..79cf8c869b94c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/DeduplicateTest.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableTestBase} import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LegacySinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LegacySinkTest.scala index 1f72b868d8c16..599e4e9207717 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LegacySinkTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LegacySinkTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{ExplainDetail, TableException} import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.types.logical.{BigIntType, IntType, VarCharType} + import org.junit.Test class LegacySinkTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LimitTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LimitTest.scala index 020e1f50932ae..d8e7287d3a397 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LimitTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/LimitTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.SqlParserException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/MiniBatchIntervalInferTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/MiniBatchIntervalInferTest.scala index fa94d4e06b254..849cc83d2d5c4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/MiniBatchIntervalInferTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/MiniBatchIntervalInferTest.scala @@ -20,13 +20,13 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableConfig +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_EARLY_FIRE_DELAY, TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED} import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} import org.apache.flink.table.types.logical.{BigIntType, IntType, VarCharType} + import org.junit.{Before, Test} class MiniBatchIntervalInferTest extends TableTestBase { @@ -390,7 +390,7 @@ class MiniBatchIntervalInferTest extends TableTestBase { throw new RuntimeException("Currently not support different earlyFireInterval configs in " + "one job") } - tableConfig.getConfiguration.setBoolean(TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED, true) + tableConfig.getConfiguration.setBoolean(TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED, Boolean.box(true)) tableConfig.getConfiguration.setString( TABLE_EXEC_EMIT_EARLY_FIRE_DELAY, intervalInMillis + " ms") } diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/ModifiedMonotonicityTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/ModifiedMonotonicityTest.scala index a87c3644b3a7c..1c7803514ddae 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/ModifiedMonotonicityTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/ModifiedMonotonicityTest.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.plan.`trait`.RelModifiedMonotonicity import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PartitionableSinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PartitionableSinkTest.scala index 4b1a0386b3372..e8b5c461aed17 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PartitionableSinkTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/PartitionableSinkTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RankTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RankTest.scala index d3a2906c037c1..37cbb21da1909 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RankTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RankTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{ExplainDetail, TableException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RelTimeIndicatorConverterTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RelTimeIndicatorConverterTest.scala index ea95b4da47384..c8f740f17c2d6 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RelTimeIndicatorConverterTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/RelTimeIndicatorConverterTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.data.TimestampData import org.apache.flink.table.functions.TableFunction import org.apache.flink.table.planner.plan.stream.sql.RelTimeIndicatorConverterTest.TableFunc diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.scala index 67ac146a960c4..4d771f4f61911 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.scala @@ -21,8 +21,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.NonPojo import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortLimitTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortLimitTest.scala index f509a5a80a7fb..5469f80bfa0bd 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortLimitTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortLimitTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortTest.scala index a4577a8b1dfd6..900ae4ef09735 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SortTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SubplanReuseTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SubplanReuseTest.scala index 225bc8c4d72e3..fa914dbef4646 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SubplanReuseTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SubplanReuseTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.functions.aggfunctions.FirstValueAggFunction.IntFirstValueAggFunction import org.apache.flink.table.planner.functions.aggfunctions.LastValueAggFunction.LongLastValueAggFunction import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.NonDeterministicUdf diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.scala index 40fad2f067852..c3b9a8dbbd2fa 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableScanTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{ExplainDetail, TableException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func0 import org.apache.flink.table.planner.factories.TestValuesTableFactory.{MockedFilterPushDownTableSource, MockedLookupTableSource} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala index f8e742e1d01f7..b5d31362acb87 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, ExplainDetail, TableException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.types.logical.LogicalType + import org.junit.Test class TableSinkTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/UnionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/UnionTest.scala index 330925886b07a..1535de46b106a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/UnionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/UnionTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/AggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/AggregateTest.scala index 730ece932968c..f01fad76d7b5c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/AggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/AggregateTest.scala @@ -20,9 +20,8 @@ package org.apache.flink.table.planner.plan.stream.sql.agg import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{ExplainDetail, TableException, Types, ValidationException} import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableTestBase} import org.apache.flink.table.runtime.typeutils.DecimalDataTypeInfo diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/DistinctAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/DistinctAggregateTest.scala index b6984082eb1a7..16316cd4084be 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/DistinctAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/DistinctAggregateTest.scala @@ -20,9 +20,8 @@ package org.apache.flink.table.planner.plan.stream.sql.agg import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.plan.rules.physical.stream.IncrementalAggregateRule import org.apache.flink.table.planner.utils.{AggregatePhaseStrategy, StreamTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala index 652b8a0346a91..62eeee9182d46 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/GroupingSetsTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql.agg import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} @@ -29,7 +28,6 @@ import org.junit.Test import java.sql.Date - class GroupingSetsTest extends TableTestBase { private val util = streamTestUtil() diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/OverAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/OverAggregateTest.scala index db39e23316ba6..89a3316b7fb1d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/OverAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/OverAggregateTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql.agg import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.OverAgg0 import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/TwoStageAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/TwoStageAggregateTest.scala index 175d2c0a36d6e..49a25f9ebf206 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/TwoStageAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/TwoStageAggregateTest.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.planner.plan.stream.sql.agg import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.{AggregatePhaseStrategy, TableTestBase} import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala index f219c46091d67..09bd5f05e39a7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala @@ -20,15 +20,13 @@ package org.apache.flink.table.planner.plan.stream.sql.agg import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{ExplainDetail, TableException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_LATE_FIRE_DELAY, TABLE_EXEC_EMIT_LATE_FIRE_ENABLED} import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test - class WindowAggregateTest extends TableTestBase { private val util = streamTestUtil() diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala index 96e103c70887c..a26034965a516 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ExplainDetail -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala index 6a3cfa2ef8c69..db0001ffcbd40 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/LookupJoinTest.scala @@ -23,7 +23,6 @@ import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ import org.apache.flink.table.data.RowData import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE import org.apache.flink.table.descriptors.{CustomConnectorDescriptor, DescriptorProperties, Schema} @@ -34,6 +33,7 @@ import org.apache.flink.table.planner.utils.TableTestBase import org.apache.flink.table.sources._ import org.apache.flink.table.types.DataType import org.apache.flink.table.utils.EncodingUtils + import org.junit.Assert.{assertTrue, fail} import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.scala index 8a53a896c37d1..5f9aa96a2d2fb 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.stream.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.StringSplit import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/TemporalJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/TemporalJoinTest.scala index 29b255d163e8e..0c79555cbdde2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/TemporalJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/TemporalJoinTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.stream.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableTestBase} import org.hamcrest.Matchers.containsString diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/WindowJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/WindowJoinTest.scala index 975024044e0ce..70ccea1265d7b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/WindowJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/WindowJoinTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql.join import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.WindowJoinUtil import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableTestBase, TableTestUtil} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/AggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/AggregateTest.scala index 0532fc47f0435..3498f7d34190a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/AggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/AggregateTest.scala @@ -20,8 +20,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.planner.utils.{CountMinMax, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala index 17840bc9a9af6..2a9d8d5cc2058 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CalcTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Tumble -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.{Func1, Func23, Func24} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/ColumnFunctionsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/ColumnFunctionsTest.scala index 119433bf5b347..24bfe9099d032 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/ColumnFunctionsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/ColumnFunctionsTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Over, Slide} +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{CountDistinct, WeightedAvg} import org.apache.flink.table.planner.utils.{CountAggFunction, TableFunc0, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CorrelateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CorrelateTest.scala index a502f4cf811b8..6875be79972b0 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CorrelateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/CorrelateTest.scala @@ -18,10 +18,11 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func13 import org.apache.flink.table.planner.plan.optimize.program.FlinkStreamProgram -import org.apache.flink.table.planner.utils.{HierarchyTableFunction, PojoTableFunc, MockPythonTableFunction, TableFunc0, TableFunc1, TableFunc2, TableTestBase} +import org.apache.flink.table.planner.utils._ + import org.apache.calcite.rel.rules.{CalcMergeRule, FilterCalcMergeRule, ProjectCalcMergeRule} import org.apache.calcite.tools.RuleSets import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTableAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTableAggregateTest.scala index 9901b1abd3a3a..ec8eeafed9ec2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTableAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTableAggregateTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Slide, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{EmptyTableAggFunc, TableTestBase} + import org.junit.Test class GroupWindowTableAggregateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTest.scala index b416170efe3ee..446d18d3a9a2f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/GroupWindowTest.scala @@ -18,15 +18,15 @@ package org.apache.flink.table.planner.plan.stream.table -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge} import org.apache.flink.table.planner.utils.{EmptyTableAggFunc, TableTestBase} + import org.junit.Test +import java.sql.Timestamp + class GroupWindowTest extends TableTestBase { @Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/JoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/JoinTest.scala index c76e630eca6c0..b80330e25e616 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/JoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/JoinTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/LegacyTableSourceTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/LegacyTableSourceTest.scala index ef79c5e00622a..62112cf0bf493 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/LegacyTableSourceTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/LegacyTableSourceTest.scala @@ -20,10 +20,9 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Over, TableSchema, Tumble, Types} -import org.apache.flink.table.planner.utils.{TableTestBase, TestNestedProjectableTableSource, TestLegacyProjectableTableSource, TestTableSourceWithTime} +import org.apache.flink.table.planner.utils.{TableTestBase, TestLegacyProjectableTableSource, TestNestedProjectableTableSource, TestTableSourceWithTime} import org.apache.flink.types.Row import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/OverWindowTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/OverWindowTest.scala index b6b912574cfe1..ae96a9a707547 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/OverWindowTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/OverWindowTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Over, Table} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func1 import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithRetract import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/PythonCalcTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/PythonCalcTest.scala index d25b20fb4afd5..f6e1c8e58a2ab 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/PythonCalcTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/PythonCalcTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.planner.utils.TableTestBase + import org.junit.{Before, Test} class PythonCalcTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/SetOperatorsTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/SetOperatorsTest.scala index b9101871a36f0..262ba24bd2bff 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/SetOperatorsTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableAggregateTest.scala index 58cabfde1a4a6..caf53a8d1c67f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableAggregateTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func0 import org.apache.flink.table.planner.utils.{EmptyTableAggFunc, EmptyTableAggFuncWithIntResultType, TableTestBase} + import org.junit.Test class TableAggregateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala index 0f0ead91fcfc7..c51801e2c783a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TableSourceTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.stream.table -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Over, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.{Ignore, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TemporalTableJoinTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TemporalTableJoinTest.scala index 81fb0485a3fdd..f2a0347274458 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TemporalTableJoinTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TemporalTableJoinTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, Table, TableSchema, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.{Expression, FieldReferenceExpression} import org.apache.flink.table.functions.{TemporalTableFunction, TemporalTableFunctionImpl} import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TwoStageAggregateTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TwoStageAggregateTest.scala index cc26e5207ad7a..1ff53702475fc 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TwoStageAggregateTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/TwoStageAggregateTest.scala @@ -20,9 +20,8 @@ package org.apache.flink.table.planner.plan.stream.table import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.DataTypes +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.{ExecutionConfigOptions, OptimizerConfigOptions} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.utils.{AggregatePhaseStrategy, StreamTableTestUtil, TableTestBase} import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/AggregateStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/AggregateStringExpressionTest.scala index 733202d96c304..933cc66d0de77 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/AggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/AggregateStringExpressionTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Tumble -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMergeAndReset} import org.apache.flink.table.planner.utils.{CountAggFunction, CountMinMax, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CalcStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CalcStringExpressionTest.scala index 6164dfe27e1bf..e894cb774eb42 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CalcStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CalcStringExpressionTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func23 import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CorrelateStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CorrelateStringExpressionTest.scala index 512c813fe2a69..461b2bbde4b07 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CorrelateStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/CorrelateStringExpressionTest.scala @@ -24,7 +24,7 @@ import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream} import org.apache.flink.streaming.api.scala.DataStream import org.apache.flink.table.api.Expressions.$ import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils._ import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowStringExpressionTest.scala index da73359ec8478..09cc8185419b2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{Session, Slide, Tumble} import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.planner.utils.{CountAggFunction, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala index c426d182fd222..c263d11196d40 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{TableTestBase, Top3} + import org.junit.Test class GroupWindowTableAggregateStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/OverWindowStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/OverWindowStringExpressionTest.scala index de916e72d0892..2db10d5c72dfb 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/OverWindowStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/OverWindowStringExpressionTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Over -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.Func1 import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithRetract} import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala index 2044de09a7ed8..6c7576c7ef7fd 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/TableAggregateStringExpressionTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/TableAggregateStringExpressionTest.scala index 0e4c9563ee105..d32f5829e2d59 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/TableAggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/stringexpr/TableAggregateStringExpressionTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.expressions.utils.Func0 import org.apache.flink.table.planner.utils.{EmptyTableAggFunc, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala index e184be88b1389..5d5d838ffedaf 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/AggregateValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{TableFunc0, TableTestBase} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CalcValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CalcValidationTest.scala index b2ab11ed61133..876737282d60a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CalcValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CalcValidationTest.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Tumble, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.planner.utils.{TableFunc0, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CorrelateValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CorrelateValidationTest.scala index 6432d45ab2ec4..f7edd6aad3f05 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CorrelateValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/CorrelateValidationTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.expressions.utils._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.planner.utils.{ObjectTableFunction, TableFunc1, TableFunc2, TableTestBase} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowTableAggregateValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowTableAggregateValidationTest.scala index 1f619253b96e4..c7c5a3becd086 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowTableAggregateValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowTableAggregateValidationTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Slide, TableException, Tumble, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_EARLY_FIRE_DELAY, TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.planner.utils.{TableTestBase, Top3} + import org.junit.Test class GroupWindowTableAggregateValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowValidationTest.scala index 418ffe8d616b0..2a84a1a5868b1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/GroupWindowValidationTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{Session, Slide, Tumble, ValidationException} import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.planner.utils.TableTestBase diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/LegacyTableSinkValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/LegacyTableSinkValidationTest.scala index 51193f68513d9..ce0293cf364cc 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/LegacyTableSinkValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/LegacyTableSinkValidationTest.scala @@ -21,12 +21,13 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, TableException, TableSchema, ValidationException} import org.apache.flink.table.planner.runtime.utils.{TableEnvUtil, TestData, TestingAppendSink, TestingUpsertTableSink} import org.apache.flink.table.planner.utils.{MemoryTableSourceSinkUtil, TableTestBase, TableTestUtil} import org.apache.flink.types.Row + import org.junit.Test class LegacyTableSinkValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/OverWindowValidationTest.scala index 87b39686521a0..6905cb0ea21c9 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/OverWindowValidationTest.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Over, Table, Tumble, ValidationException} import org.apache.flink.table.planner.delegation.PlannerBase import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvgWithRetract import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.OverAgg0 diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/SetOperatorsValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/SetOperatorsValidationTest.scala index cb16186d7ac15..804050ce44e88 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/SetOperatorsValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/SetOperatorsValidationTest.scala @@ -20,11 +20,12 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.{TestData, TestingAppendSink} import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TableAggregateValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TableAggregateValidationTest.scala index 918fbe1620db6..ee29eed7db1dc 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TableAggregateValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TableAggregateValidationTest.scala @@ -21,8 +21,7 @@ package org.apache.flink.table.planner.plan.stream.table.validation import java.sql.Timestamp import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{EmptyTableAggFunc, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TemporalTableJoinValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TemporalTableJoinValidationTest.scala index d9d35d06ed4a1..77e980530b47f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TemporalTableJoinValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/TemporalTableJoinValidationTest.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Table, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.utils.{TableTestBase, TableTestUtil} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/UnsupportedOpsValidationTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/UnsupportedOpsValidationTest.scala index 91a7eb6ed57cd..484b1d3881ef1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/UnsupportedOpsValidationTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/stream/table/validation/UnsupportedOpsValidationTest.scala @@ -20,11 +20,12 @@ package org.apache.flink.table.planner.plan.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.TestData import org.apache.flink.table.planner.utils.TableTestUtil import org.apache.flink.test.util.AbstractTestBase + import org.junit.Test class UnsupportedOpsValidationTest extends AbstractTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtilTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtilTest.scala index d1f4b1040b090..f78032443f8c8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtilTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtilTest.scala @@ -21,9 +21,9 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo.{DOUBLE_TYPE_INFO, INT import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api.bridge.scala.{StreamTableEnvironment, _} import org.apache.flink.table.api.internal.TableEnvironmentImpl -import org.apache.flink.table.api.scala.{StreamTableEnvironment, _} -import org.apache.flink.table.api.{EnvironmentSettings, TableEnvironment} +import org.apache.flink.table.api.{EnvironmentSettings, TableEnvironment, _} import org.apache.flink.table.planner.plan.`trait`.{MiniBatchInterval, MiniBatchMode} import org.apache.flink.table.planner.runtime.utils.BatchTableEnvUtil import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/AggregationITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/AggregationITCase.scala index e8f10638cefa4..54e62d79a7390 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/AggregationITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/AggregationITCase.scala @@ -21,8 +21,7 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2} import org.apache.flink.api.java.typeutils.{ObjectArrayTypeInfo, TupleTypeInfo} import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.AggregateFunction import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{CountDistinctWithMergeAndReset, WeightedAvgWithMergeAndReset} import org.apache.flink.table.planner.runtime.utils.{BatchTableEnvUtil, BatchTestBase, CollectionBatchExecTable} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CalcITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CalcITCase.scala index c1ee15e4a7eb4..f948eb38543b8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CalcITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CalcITCase.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.scala._ import org.apache.flink.table.api.DataTypes._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.data.DecimalDataUtils import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.expressions.utils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CorrelateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CorrelateITCase.scala index e39f5df9baa66..da4727a13702b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CorrelateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/CorrelateITCase.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, Table, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.{Func1, Func18, FuncWithOpen, RichFunc2} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.JavaTableFunc0 import org.apache.flink.table.planner.runtime.utils.{BatchTableEnvUtil, BatchTestBase, CollectionBatchExecTable, UserDefinedFunctionTestUtils} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/DecimalITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/DecimalITCase.scala index aefa6c606d3dd..6ba2f72f70468 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/DecimalITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/DecimalITCase.scala @@ -19,9 +19,8 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, Table} import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils.{BatchTableEnvUtil, BatchTestBase} import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.fromDataTypeToLogicalType diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/GroupWindowITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/GroupWindowITCase.scala index 226c702545b67..112cd7cb81272 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/GroupWindowITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/GroupWindowITCase.scala @@ -19,8 +19,7 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, TableException, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.{BatchTableEnvUtil, BatchTestBase} import org.apache.flink.table.planner.utils.CountAggFunction import org.apache.flink.test.util.TestBaseUtils diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/JoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/JoinITCase.scala index 43ddbc26108b0..75506a9b398e1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/JoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/JoinITCase.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.expressions.utils.FuncWithOpen import org.apache.flink.table.planner.runtime.batch.sql.join.JoinITCaseHelper.disableOtherJoinOpForJoin import org.apache.flink.table.planner.runtime.batch.sql.join.JoinType diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/LegacyTableSinkITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/LegacyTableSinkITCase.scala index 85e2b964bf1c3..002b8256d99c1 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/LegacyTableSinkITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/LegacyTableSinkITCase.scala @@ -18,17 +18,17 @@ package org.apache.flink.table.planner.runtime.batch.table -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, TableSchema} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.runtime.utils.TestData._ import org.apache.flink.table.planner.runtime.utils.{BatchTestBase, TestingRetractTableSink, TestingUpsertTableSink} import org.apache.flink.table.planner.utils.MemoryTableSourceSinkUtil import org.apache.flink.test.util.TestBaseUtils + import org.junit.Assert._ import org.junit._ -import java.util.TimeZone -import org.apache.flink.table.api.internal.TableEnvironmentInternal +import java.util.TimeZone import scala.collection.JavaConverters._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/OverWindowITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/OverWindowITCase.scala index 97a2bc547e0eb..bb622f6a975a5 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/OverWindowITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/OverWindowITCase.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.runtime.batch.table -import org.apache.flink.table.api.Over -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{Over, _} import org.apache.flink.table.planner.runtime.utils.BatchTestBase import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils.TestData._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SetOperatorsITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SetOperatorsITCase.scala index b1350b2f62df1..93b7ffed1d84e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SetOperatorsITCase.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.runtime.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.batch.sql.join.JoinITCaseHelper.disableOtherJoinOpForJoin import org.apache.flink.table.planner.runtime.batch.sql.join.JoinType import org.apache.flink.table.planner.runtime.batch.sql.join.JoinType.JoinType diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SortITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SortITCase.scala index a6403e4046a4b..8b1bcd5463331 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SortITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/SortITCase.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.runtime.batch.table -import org.apache.flink.table.api.Table -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.runtime.utils.SortTestUtils.{sortExpectedly, tupleDataSetStrings} import org.apache.flink.table.planner.runtime.utils.{BatchTestBase, CollectionBatchExecTable} import org.apache.flink.test.util.TestBaseUtils diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/TableSinkITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/TableSinkITCase.scala index eef1c463e72a7..f6aaf0f2a393f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/TableSinkITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/table/TableSinkITCase.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.planner.runtime.batch.table -import org.apache.flink.table.api.DataTypes -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.BatchTestBase import org.apache.flink.table.planner.runtime.utils.TestData._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/GroupAggregateHarnessTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/GroupAggregateHarnessTest.scala index 411d73fd1cd06..4aa0715ff7b13 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/GroupAggregateHarnessTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/GroupAggregateHarnessTest.scala @@ -20,8 +20,9 @@ package org.apache.flink.table.planner.runtime.harness import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.api.{EnvironmentSettings, Types} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.runtime.util.RowDataHarnessAssertor diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/OverWindowHarnessTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/OverWindowHarnessTest.scala index 216d55976c3d3..54c74dd5b4d66 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/OverWindowHarnessTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/OverWindowHarnessTest.scala @@ -22,13 +22,14 @@ import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.runtime.streamrecord.StreamRecord -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl -import org.apache.flink.table.api.{EnvironmentSettings, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.runtime.util.RowDataHarnessAssertor import org.apache.flink.table.runtime.util.StreamRecordUtils.{binaryrow, row} import org.apache.flink.types.Row + import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/TableAggregateHarnessTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/TableAggregateHarnessTest.scala index dfb2d730f0ca1..ae050863f44cb 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/TableAggregateHarnessTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/harness/TableAggregateHarnessTest.scala @@ -22,8 +22,9 @@ import java.lang.{Integer => JInt} import java.util.concurrent.ConcurrentLinkedQueue import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.api.{EnvironmentSettings, Types} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.utils.{Top3WithMapView, Top3WithRetractInput} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala index 58867cbbf18ca..73b5801a2673f 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateITCase.scala @@ -24,8 +24,9 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{Types, _} +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.functions.aggfunctions.{ListAggWithRetractAggFunction, ListAggWsWithRetractAggFunction} import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.VarSumAggFunction @@ -39,15 +40,15 @@ import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.table.planner.utils.DateTimeTestUtil.{localDate, localDateTime, localTime => mLocalTime} import org.apache.flink.table.runtime.typeutils.BigDecimalTypeInfo import org.apache.flink.types.{Row, RowKind} + import org.junit.Assert.assertEquals import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized + import java.lang.{Integer => JInt, Long => JLong} import java.math.{BigDecimal => JBigDecimal} -import org.apache.flink.table.api.internal.TableEnvironmentInternal - import scala.collection.{Seq, mutable} import scala.util.Random diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateRemoveITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateRemoveITCase.scala index abe4e7866c648..f18876fef3587 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateRemoveITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AggregateRemoveITCase.scala @@ -18,7 +18,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils.StreamingWithAggTestBase.AggMode diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AsyncLookupJoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AsyncLookupJoinITCase.scala index ef3fd8dd66a97..cbdcecbd47fca 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AsyncLookupJoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/AsyncLookupJoinITCase.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ import org.apache.flink.table.api.{TableSchema, Types} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.{HEAP_BACKEND, ROCKSDB_BACKEND, StateBackendMode} import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala index d32c9ea05e155..3771acc51de39 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CalcITCase.scala @@ -22,8 +22,9 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.api.scala.typeutils.Types +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.data.{GenericRowData, RowData} import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row @@ -31,6 +32,7 @@ import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.table.runtime.typeutils.RowDataTypeInfo import org.apache.flink.table.types.logical.{BigIntType, IntType, VarCharType} import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CorrelateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CorrelateITCase.scala index 6aff49e1ab5e3..762c2e624ab87 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CorrelateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/CorrelateITCase.scala @@ -18,18 +18,19 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.UdfWithOpen import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedTableFunctions.StringSplit import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestSinkUtil, TestingAppendSink, TestingAppendTableSink} import org.apache.flink.table.planner.utils.{RF, TableFunc7} import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.{Before, Test} -import java.lang.{Boolean => JBoolean} -import org.apache.flink.table.api.internal.TableEnvironmentInternal +import java.lang.{Boolean => JBoolean} import scala.collection.mutable diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/DeduplicateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/DeduplicateITCase.scala index a0f5b2cf93be9..4f192f6e14b6c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/DeduplicateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/DeduplicateITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithMiniBatchTestBase.MiniBatchMode import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/JoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/JoinITCase.scala index 36e8a45cd2f84..947fd532db1f7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/JoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/JoinITCase.scala @@ -20,7 +20,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.expressions.utils.FuncWithOpen import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LegacyTableSourceITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LegacyTableSourceITCase.scala index a15ae3605ee9e..bee95269f31f4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LegacyTableSourceITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LegacyTableSourceITCase.scala @@ -22,7 +22,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{DataTypes, TableSchema, Types} import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestData, TestingAppendSink} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/Limit0RemoveITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/Limit0RemoveITCase.scala index 4c61f608c7cbf..ab0481db02817 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/Limit0RemoveITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/Limit0RemoveITCase.scala @@ -19,9 +19,11 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestSinkUtil, TestingAppendTableSink, TestingRetractTableSink} + import org.junit.Assert.assertEquals import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LimitITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LimitITCase.scala index 4120bec1314a4..9d1e732fa3ba4 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LimitITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LimitITCase.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{TableException, _} +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LookupJoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LookupJoinITCase.scala index 449f63a1673f7..142409a413a4b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LookupJoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/LookupJoinITCase.scala @@ -18,12 +18,13 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableSchema, Types} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils.TestAddWithOpen import org.apache.flink.table.planner.runtime.utils.{InMemoryLookupableTableSource, StreamingTestBase, TestingAppendSink} import org.apache.flink.types.Row + import org.junit.Assert.{assertEquals, assertTrue} import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/MatchRecognizeITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/MatchRecognizeITCase.scala index b9b91987b14cb..787f062dcd480 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/MatchRecognizeITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/MatchRecognizeITCase.scala @@ -23,8 +23,8 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.Types +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.{AggregateFunction, FunctionContext, ScalarFunction} import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode @@ -32,6 +32,7 @@ import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.EventTimeSource import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingAppendSink, UserDefinedFunctionTestUtils} import org.apache.flink.table.planner.utils.TableTestUtil import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/OverWindowITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/OverWindowITCase.scala index 2e8cf75bf283d..b1f5bf809f552 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/OverWindowITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/OverWindowITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.EventTimeProcessOperator import org.apache.flink.table.planner.runtime.utils.UserDefinedFunctionTestUtils.{CountNullNonNull, CountPairs, LargerThanCount} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/PruneAggregateCallITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/PruneAggregateCallITCase.scala index 3c588c9e61d11..657c8dd293960 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/PruneAggregateCallITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/PruneAggregateCallITCase.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils.StreamingWithAggTestBase.AggMode diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/RankITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/RankITCase.scala index a422ed822ff33..75fae89bd3a56 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/RankITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/RankITCase.scala @@ -21,16 +21,18 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode -import org.apache.flink.table.planner.runtime.utils.{TestingRetractTableSink, TestingUpsertTableSink, _} +import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.table.runtime.types.TypeInfoDataTypeConverter.fromDataTypeToTypeInfo import org.apache.flink.types.Row + import org.junit.Assert._ +import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized -import org.junit.{Ignore, _} @RunWith(classOf[Parameterized]) class RankITCase(mode: StateBackendMode) extends StreamingWithStateTestBase(mode) { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SemiAntiJoinStreamITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SemiAntiJoinStreamITCase.scala index 0cb9a477075f8..d1d24ec1f7413 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SemiAntiJoinStreamITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SemiAntiJoinStreamITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestData, TestingRetractSink} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SetOperatorsITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SetOperatorsITCase.scala index ce8414da02532..00e7627f2add0 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SetOperatorsITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestData, TestingRetractSink} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortITCase.scala index 6644883017799..6df256172f35e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortITCase.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamExecSort import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortLimitITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortLimitITCase.scala index ab68913be89ce..09573d2f01914 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortLimitITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SortLimitITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SplitAggregateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SplitAggregateITCase.scala index e59e42cc14c67..c1c34c7e527ff 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SplitAggregateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/SplitAggregateITCase.scala @@ -20,9 +20,9 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.config.OptimizerConfigOptions -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.runtime.stream.sql.SplitAggregateITCase.PartialAggMode import org.apache.flink.table.planner.runtime.utils.StreamingWithAggTestBase.{AggMode, LocalGlobalOff, LocalGlobalOn} import org.apache.flink.table.planner.runtime.utils.StreamingWithMiniBatchTestBase.MiniBatchOn diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamFileSystemITCaseBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamFileSystemITCaseBase.scala index c32379f6b32f9..2a61be84becdb 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamFileSystemITCaseBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamFileSystemITCaseBase.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ import org.apache.flink.table.api.TableEnvironment -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.FileSystemITCaseBase import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestSinkUtil, TestingAppendSink} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala index d26c86484ea00..115e5bb55b3f5 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala @@ -20,7 +20,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.JavaPojos.{Device, Order, ProductItem} import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, StringSink} @@ -30,7 +31,7 @@ import org.junit.Test import java.util.Collections /** - * Integration tests for methods on [[org.apache.flink.table.api.scala.StreamTableEnvironment]]. + * Integration tests for methods on [[StreamTableEnvironment]]. */ class StreamTableEnvironmentITCase extends StreamingTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableScanITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableScanITCase.scala index 34ab3154bdd1c..839932db88a6a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableScanITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableScanITCase.scala @@ -20,19 +20,20 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.functions.ProcessFunction -import org.apache.flink.streaming.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableSchema, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestingAppendSink} import org.apache.flink.table.planner.utils.{TestPreserveWMTableSource, TestTableSourceWithTime, WithoutTimeAttributesTableSource} import org.apache.flink.types.Row import org.apache.flink.util.Collector + import org.junit.Assert._ import org.junit.Test -import java.lang.{Integer => JInt, Long => JLong} -import org.apache.flink.table.api.internal.TableEnvironmentInternal +import java.lang.{Integer => JInt, Long => JLong} class TableScanITCase extends StreamingTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableSourceITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableSourceITCase.scala index 6e0964c576610..a6fc483f0fb9c 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableSourceITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TableSourceITCase.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestData, TestingAppendSink, TestingRetractSink} import org.apache.flink.table.planner.utils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalJoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalJoinITCase.scala index 4e03680c1b5a2..3d3d909f15f25 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalJoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalJoinITCase.scala @@ -23,11 +23,13 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.windowing.time.Time -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingAppendSink} import org.apache.flink.table.planner.utils.TableTestUtil import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit._ import org.junit.runner.RunWith diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalSortITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalSortITCase.scala index e34152fc69d5c..de90d7ee670da 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalSortITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TemporalSortITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.TimestampAndWatermarkWithOffset import org.apache.flink.table.planner.runtime.utils._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimeAttributeITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimeAttributeITCase.scala index 311c682d0f962..48fffed64bb26 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimeAttributeITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimeAttributeITCase.scala @@ -19,12 +19,13 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.JavaFunc5 import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestingAppendSink} import org.apache.flink.types.Row + import org.junit.Assert.{assertEquals, assertTrue} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimestampITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimestampITCase.scala index ce3bb6469e1b3..61043af7b8c19 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimestampITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/TimestampITCase.scala @@ -19,13 +19,14 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, TableSchema} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.BatchTestBase.row import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestingRetractSink} import org.apache.flink.table.planner.utils.DateTimeTestUtil.localDateTime import org.apache.flink.table.planner.utils.TestDataTypeTableSourceWithTime import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/UnnestITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/UnnestITCase.scala index 06ebe4cc03178..93f802bd6e22a 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/UnnestITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/UnnestITCase.scala @@ -20,13 +20,14 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.TimestampAndWatermarkWithOffset -import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestData, TestingAppendSink, TestingRetractSink, TestingRetractTableSink} +import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/ValuesITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/ValuesITCase.scala index 1d2ea7184b844..54fc36bba866d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/ValuesITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/ValuesITCase.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.planner.runtime.stream.sql import org.apache.flink.streaming.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.data.RowData import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, TestingAppendRowDataSink} import org.apache.flink.table.runtime.typeutils.RowDataTypeInfo import org.apache.flink.table.types.logical.{IntType, VarCharType} + import org.junit.Assert._ import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala index decefaa704115..7d3d8ec9df1ed 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala @@ -18,12 +18,12 @@ package org.apache.flink.table.planner.runtime.stream.sql - import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableConfig, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{ConcatDistinctAggFunction, WeightedAvg} import org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_LATE_FIRE_DELAY, TABLE_EXEC_EMIT_LATE_FIRE_ENABLED} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode @@ -31,15 +31,15 @@ import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.TimestampAndWat import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.table.planner.utils.TableConfigUtils.getMillisecondFromConfigDuration import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized + import java.math.BigDecimal import java.util.concurrent.TimeUnit -import org.apache.flink.table.api.internal.TableEnvironmentInternal - @RunWith(classOf[Parameterized]) class WindowAggregateITCase(mode: StateBackendMode) extends StreamingWithStateTestBase(mode) { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowJoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowJoinITCase.scala index 6d0488ef48d9e..b0840fc6eb486 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowJoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowJoinITCase.scala @@ -22,7 +22,8 @@ import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/AggregateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/AggregateITCase.scala index 10dbef6c69ca8..bb9bba03fabaa 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/AggregateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/AggregateITCase.scala @@ -21,15 +21,16 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, DataViewTestAgg, WeightedAvg} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TestData._ import org.apache.flink.table.planner.runtime.utils.{JavaUserDefinedAggFunctions, StreamingWithStateTestBase, TestingRetractSink, TestingUpsertTableSink} import org.apache.flink.table.planner.utils.CountMinMax import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CalcITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CalcITCase.scala index a0321a027e641..f5dc5db24adf0 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CalcITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CalcITCase.scala @@ -19,14 +19,15 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.planner.expressions.utils._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TestData._ import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingAppendSink, TestingRetractSink, UserDefinedFunctionTestUtils} import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit._ import org.junit.runner.RunWith diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CorrelateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CorrelateITCase.scala index 61b432e8aba30..8ae73c7c5712e 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CorrelateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/CorrelateITCase.scala @@ -19,13 +19,13 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Types, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.expressions.utils.{Func18, FuncWithOpen, RichFunc2} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TestData._ import org.apache.flink.table.planner.runtime.utils._ -import org.apache.flink.table.planner.utils.{PojoTableFunc, RF, RichTableFunc1, TableFunc0, TableFunc2, TableFunc3, TableFunc6, TableFunc7, VarArgsFunc0} +import org.apache.flink.table.planner.utils._ import org.apache.flink.types.Row import org.junit.Assert._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowITCase.scala index 031f1d6f42c34..5f4be8ab0233d 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithMerge, WeightedAvg, WeightedAvgWithMerge} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.TimestampAndWatermarkWithOffset diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowTableAggregateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowTableAggregateITCase.scala index 456fa3276f925..85f18187c424b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowTableAggregateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/GroupWindowTableAggregateITCase.scala @@ -18,23 +18,24 @@ package org.apache.flink.table.planner.runtime.stream.table -import java.math.BigDecimal - import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode +import org.apache.flink.table.planner.runtime.utils.TestData._ import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.TimestampAndWatermarkWithOffset import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingAppendSink} import org.apache.flink.table.planner.utils.Top3 import org.apache.flink.types.Row -import org.apache.flink.table.planner.runtime.utils.TestData._ + import org.junit.Assert._ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.math.BigDecimal + @RunWith(classOf[Parameterized]) class GroupWindowTableAggregateITCase(mode: StateBackendMode) extends StreamingWithStateTestBase(mode) { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/JoinITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/JoinITCase.scala index d82f02ca07cc0..c34c5a42606f2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/JoinITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/JoinITCase.scala @@ -21,16 +21,17 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Tumble, Types} import org.apache.flink.table.planner.expressions.utils.FuncWithOpen import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, WeightedAvg} import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TestData._ -import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingAppendSink, TestingRetractSink, TestingRetractTableSink, TestingUpsertTableSink} +import org.apache.flink.table.planner.runtime.utils._ import org.apache.flink.table.planner.utils.CountAggFunction import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/LegacyTableSinkITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/LegacyTableSinkITCase.scala index b0fd938fea547..bdb121ac49e1b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/LegacyTableSinkITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/LegacyTableSinkITCase.scala @@ -22,21 +22,22 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, TableException, TableSchema, Tumble, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.planner.runtime.utils.TestData.{smallTupleData3, tupleData3, tupleData5} import org.apache.flink.table.planner.runtime.utils.{TableEnvUtil, TestingAppendTableSink, TestingRetractTableSink, TestingUpsertTableSink} import org.apache.flink.table.planner.utils.{MemoryTableSourceSinkUtil, TableTestUtil} import org.apache.flink.table.sinks._ import org.apache.flink.test.util.{AbstractTestBase, TestBaseUtils} import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit.Test + import java.io.File import java.util.TimeZone -import org.apache.flink.table.api.internal.TableEnvironmentInternal - import scala.collection.JavaConverters._ class LegacyTableSinkITCase extends AbstractTestBase { diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/MiniBatchGroupWindowITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/MiniBatchGroupWindowITCase.scala index 3ade387621823..9b7f903472efe 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/MiniBatchGroupWindowITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/MiniBatchGroupWindowITCase.scala @@ -18,8 +18,8 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Slide, Tumble} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, WeightedAvg} import org.apache.flink.table.planner.runtime.utils.StreamingWithMiniBatchTestBase.MiniBatchMode import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/OverWindowITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/OverWindowITCase.scala index 9d43bae32d57d..b3a5c1c92f0a3 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/OverWindowITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/OverWindowITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, Expressions, Over} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithRetractAndReset, WeightedAvg} import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctions.JavaFunc0 import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/RetractionITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/RetractionITCase.scala index 64c40044f5652..5169b69fa3572 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/RetractionITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/RetractionITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingRetractSink} import org.apache.flink.table.planner.utils.TableFunc0 diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SetOperatorsITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SetOperatorsITCase.scala index 277de23d85df2..6fb7279a34450 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SetOperatorsITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.plan.utils.NonPojo import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TestData._ diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SubQueryITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SubQueryITCase.scala index f49fc232e0e25..08f808d47b016 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SubQueryITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/SubQueryITCase.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingRetractSink} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableAggregateITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableAggregateITCase.scala index 74355ea30fc8f..a2e4034de7d9b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableAggregateITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableAggregateITCase.scala @@ -20,13 +20,14 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.StateBackendMode import org.apache.flink.table.planner.runtime.utils.TestData.tupleData3 import org.apache.flink.table.planner.runtime.utils.{StreamingWithStateTestBase, TestingRetractSink} import org.apache.flink.table.planner.utils.{TableAggSum, Top3, Top3WithMapView, Top3WithRetractInput} import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableSinkITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableSinkITCase.scala index 3a724e7e0ebc0..c8d67022893a7 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableSinkITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/table/TableSinkITCase.scala @@ -19,13 +19,14 @@ package org.apache.flink.table.planner.runtime.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, TableException, Tumble} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.planner.factories.TestValuesTableFactory import org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow import org.apache.flink.table.planner.runtime.utils.StreamingTestBase import org.apache.flink.table.planner.runtime.utils.TestData.{nullData4, smallTupleData3, tupleData3, tupleData5} import org.apache.flink.util.ExceptionUtils + import org.junit.Assert.{assertEquals, assertFalse, assertTrue, fail} import org.junit.Test diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTableEnvUtil.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTableEnvUtil.scala index f6fc09ac18cef..27dd611918ee8 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTableEnvUtil.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTableEnvUtil.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.planner.runtime.utils import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.table.api.TableEnvironment -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment import org.apache.flink.table.expressions.{Expression, ExpressionParser} import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.utils.TableTestUtil diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala index 3fffbc0f15c7e..4c87d95d8a2ba 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingTestBase.scala @@ -21,10 +21,9 @@ package org.apache.flink.table.planner.runtime.utils import org.apache.flink.api.common.JobExecutionResult import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala.StreamTableEnvironment -import org.apache.flink.table.api.ImplicitExpressionConversions +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment +import org.apache.flink.table.api.{EnvironmentSettings, ImplicitExpressionConversions, Table} import org.apache.flink.table.planner.factories.TestValuesTableFactory -import org.apache.flink.table.api.{EnvironmentSettings, Table} import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingWithStateTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingWithStateTestBase.scala index f443a0c09e6ff..3182baaf48da6 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingWithStateTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamingWithStateTestBase.scala @@ -27,7 +27,7 @@ import org.apache.flink.runtime.state.memory.MemoryStateBackend import org.apache.flink.streaming.api.CheckpointingMode import org.apache.flink.streaming.api.functions.source.FromElementsFunction import org.apache.flink.streaming.api.scala.DataStream -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment import org.apache.flink.table.data.binary.BinaryRowData import org.apache.flink.table.data.writer.BinaryRowWriter import org.apache.flink.table.data.{RowData, StringData} diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala index f4b6e5b838d4b..05b54c7228f40 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala @@ -28,10 +28,10 @@ import org.apache.flink.streaming.api.{TimeCharacteristic, environment} import org.apache.flink.table.api._ import org.apache.flink.table.api.config.ExecutionConfigOptions import org.apache.flink.table.api.internal.{TableEnvironmentImpl, TableEnvironmentInternal, TableImpl} -import org.apache.flink.table.api.java.internal.{StreamTableEnvironmentImpl => JavaStreamTableEnvImpl} -import org.apache.flink.table.api.java.{StreamTableEnvironment => JavaStreamTableEnv} -import org.apache.flink.table.api.scala.internal.{StreamTableEnvironmentImpl => ScalaStreamTableEnvImpl} -import org.apache.flink.table.api.scala.{StreamTableEnvironment => ScalaStreamTableEnv} +import org.apache.flink.table.api.bridge.java.internal.{StreamTableEnvironmentImpl => JavaStreamTableEnvImpl} +import org.apache.flink.table.api.bridge.java.{StreamTableEnvironment => JavaStreamTableEnv} +import org.apache.flink.table.api.bridge.scala.internal.{StreamTableEnvironmentImpl => ScalaStreamTableEnvImpl} +import org.apache.flink.table.api.bridge.scala.{StreamTableEnvironment => ScalaStreamTableEnv} import org.apache.flink.table.catalog.{CatalogManager, FunctionCatalog, GenericInMemoryCatalog, ObjectIdentifier} import org.apache.flink.table.data.RowData import org.apache.flink.table.delegation.{Executor, ExecutorFactory, PlannerFactory} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/java/package-info.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/bridge/java/package-info.java similarity index 77% rename from flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/java/package-info.java rename to flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/bridge/java/package-info.java index 50d41a2c9ff6a..a7d7045ed9e8c 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/java/package-info.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/api/bridge/java/package-info.java @@ -19,9 +19,9 @@ /** * Table API (Java)
* - * A {@link org.apache.flink.table.api.java.BatchTableEnvironment} can be used to create a + * A {@link org.apache.flink.table.api.bridge.java.BatchTableEnvironment} can be used to create a * {@link org.apache.flink.table.api.Table} from a {@link org.apache.flink.api.java.DataSet}. - * Equivalently, a {@link org.apache.flink.table.api.java.StreamTableEnvironment} can be used to + * Equivalently, a {@link org.apache.flink.table.api.bridge.java.StreamTableEnvironment} can be used to * create a {@link org.apache.flink.table.api.Table} from a * {@link org.apache.flink.streaming.api.datastream.DataStream}. * @@ -57,11 +57,11 @@ *

* As seen above, a {@link org.apache.flink.table.api.Table} can be converted back to the * underlying API representation using - * {@link org.apache.flink.table.api.java.BatchTableEnvironment#toDataSet(Table, java.lang.Class)}, - * {@link org.apache.flink.table.api.java.StreamTableEnvironment#toAppendStream(Table, java.lang.Class)}}, or - * {@link org.apache.flink.table.api.java.StreamTableEnvironment#toRetractStream(Table, java.lang.Class)}}. + * {@link org.apache.flink.table.api.bridge.java.BatchTableEnvironment#toDataSet(Table, java.lang.Class)}, + * {@link org.apache.flink.table.api.bridge.java.StreamTableEnvironment#toAppendStream(Table, java.lang.Class)}}, or + * {@link org.apache.flink.table.api.bridge.java.StreamTableEnvironment#toRetractStream(Table, java.lang.Class)}}. */ -package org.apache.flink.table.api.java; +package org.apache.flink.table.api.bridge.java; import org.apache.flink.table.api.Table; diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/executor/StreamExecutorFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/executor/StreamExecutorFactory.java index cf9f6504feebb..307220f8e50e7 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/executor/StreamExecutorFactory.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/executor/StreamExecutorFactory.java @@ -34,7 +34,7 @@ * Factory to create an implementation of {@link Executor} to use in a * {@link org.apache.flink.table.api.TableEnvironment}. The {@link org.apache.flink.table.api.TableEnvironment} * should use {@link #create(Map)} method that does not bind to any particular environment, - * whereas {@link org.apache.flink.table.api.scala.StreamTableEnvironment} should use + * whereas {@link org.apache.flink.table.api.bridge.scala.StreamTableEnvironment} should use * {@link #create(Map, StreamExecutionEnvironment)} as it is always backed by * some {@link StreamExecutionEnvironment} */ diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/java/internal/BatchTableEnvironmentImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/bridge/java/internal/BatchTableEnvironmentImpl.scala similarity index 96% rename from flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/java/internal/BatchTableEnvironmentImpl.scala rename to flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/bridge/java/internal/BatchTableEnvironmentImpl.scala index f7f58224ce182..3a99832a065a7 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/java/internal/BatchTableEnvironmentImpl.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/bridge/java/internal/BatchTableEnvironmentImpl.scala @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.java.internal +package org.apache.flink.table.api.bridge.java.internal import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.api.java.{DataSet, ExecutionEnvironment} import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.BatchTableEnvImpl -import org.apache.flink.table.api.java.BatchTableEnvironment +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment import org.apache.flink.table.catalog.CatalogManager import org.apache.flink.table.expressions.{Expression, ExpressionParser} import org.apache.flink.table.functions.{AggregateFunction, TableFunction} @@ -48,7 +48,7 @@ class BatchTableEnvironmentImpl( config, catalogManager, moduleManager) - with org.apache.flink.table.api.java.BatchTableEnvironment { + with org.apache.flink.table.api.bridge.java.BatchTableEnvironment { override def fromDataSet[T](dataSet: DataSet[T]): Table = { createTable(asQueryOperation(dataSet, None)) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/scala/internal/BatchTableEnvironmentImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/BatchTableEnvironmentImpl.scala similarity index 94% rename from flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/scala/internal/BatchTableEnvironmentImpl.scala rename to flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/BatchTableEnvironmentImpl.scala index a46484eae632a..78133b1760196 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/scala/internal/BatchTableEnvironmentImpl.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/bridge/scala/internal/BatchTableEnvironmentImpl.scala @@ -15,13 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.flink.table.api.scala.internal +package org.apache.flink.table.api.bridge.scala.internal import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.BatchTableEnvImpl -import org.apache.flink.table.api.scala.BatchTableEnvironment +import org.apache.flink.table.api.bridge.scala.BatchTableEnvironment import org.apache.flink.table.catalog.CatalogManager import org.apache.flink.table.expressions.Expression import org.apache.flink.table.functions.{AggregateFunction, TableFunction} @@ -47,7 +47,7 @@ class BatchTableEnvironmentImpl( config, catalogManager, moduleManager) - with org.apache.flink.table.api.scala.BatchTableEnvironment { + with org.apache.flink.table.api.bridge.scala.BatchTableEnvironment { override def fromDataSet[T](dataSet: DataSet[T]): Table = { createTable(asQueryOperation(dataSet.javaSet, None)) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/PlannerExpressionParserImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/PlannerExpressionParserImpl.scala index 181a91a52ec69..29c2a2d476aa0 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/PlannerExpressionParserImpl.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/PlannerExpressionParserImpl.scala @@ -52,8 +52,8 @@ class PlannerExpressionParserImpl extends PlannerExpressionParser { * Parser for expressions inside a String. This parses exactly the same expressions that * would be accepted by the Scala Expression DSL. * - * See [[org.apache.flink.table.api.scala.ImplicitExpressionConversions]] and - * [[org.apache.flink.table.api.scala.ImplicitExpressionOperations]] for the constructs + * See [[org.apache.flink.table.api.bridge.scala.ImplicitExpressionConversions]] and + * [[org.apache.flink.table.api.bridge.scala.ImplicitExpressionOperations]] for the constructs * available in the Scala Expression DSL. This parser must be kept in sync with the Scala DSL * lazy valined in the above files. */ diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/package.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/package.scala index 41e0c9f5bc384..bd72692ef2de1 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/package.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/expressions/package.scala @@ -21,8 +21,8 @@ package org.apache.flink.table * This package contains the base class of AST nodes and all the expression language AST classes. * Expression trees should not be manually constructed by users. They are implicitly constructed * from the implicit DSL conversions in - * [[org.apache.flink.table.api.scala.ImplicitExpressionConversions]] and - * [[org.apache.flink.table.api.scala.ImplicitExpressionOperations]]. For the Java API, + * [[org.apache.flink.table.api.bridge.scala.ImplicitExpressionConversions]] and + * [[org.apache.flink.table.api.bridge.scala.ImplicitExpressionOperations]]. For the Java API, * expression trees should be generated from a string parser that parses expressions and creates * AST nodes. */ diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/DummyExecutionEnvironment.java b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/DummyExecutionEnvironment.java index 8e226a601f4a7..0a88dab31868e 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/DummyExecutionEnvironment.java +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/DummyExecutionEnvironment.java @@ -28,7 +28,7 @@ import org.apache.flink.core.execution.JobClient; import org.apache.flink.core.execution.JobListener; import org.apache.flink.core.execution.PipelineExecutorServiceLoader; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import com.esotericsoftware.kryo.Serializer; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/StreamTableEnvironmentTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/StreamTableEnvironmentTest.java index 8fa30263f9e42..740c264609e82 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/StreamTableEnvironmentTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/StreamTableEnvironmentTest.java @@ -23,7 +23,7 @@ import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.junit.Test; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/PathResolutionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/PathResolutionTest.java index 31077de03e3be..3404ef7bc017c 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/PathResolutionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/PathResolutionTest.java @@ -18,7 +18,7 @@ package org.apache.flink.table.catalog; -import org.apache.flink.table.api.java.internal.StreamTableEnvironmentImpl; +import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; import org.apache.flink.table.utils.StreamTableTestUtil; import org.apache.flink.util.Preconditions; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/ViewExpansionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/ViewExpansionTest.java index 2f52217d800d9..a03649597c28f 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/ViewExpansionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/catalog/ViewExpansionTest.java @@ -21,7 +21,7 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.java.internal.StreamTableEnvironmentImpl; +import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; import org.apache.flink.table.utils.StreamTableTestUtil; import org.junit.Rule; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/JavaTableSourceITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/JavaTableSourceITCase.java index 58c21166b5d31..997be8b4f72f1 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/JavaTableSourceITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/JavaTableSourceITCase.java @@ -21,8 +21,8 @@ import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.BatchTableEnvironment; import org.apache.flink.table.runtime.utils.CommonTestData; import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase; import org.apache.flink.table.sources.BatchTableSource; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/GroupingSetsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/GroupingSetsITCase.java index 93c1372af9bd6..b1af4f871c44b 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/GroupingSetsITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/GroupingSetsITCase.java @@ -25,7 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableConfig; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase; import org.apache.flink.test.operators.util.CollectionDataSets; import org.apache.flink.test.util.TestBaseUtils; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/JavaSqlITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/JavaSqlITCase.java index e1b4c35ba2f53..7c6cac273f483 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/JavaSqlITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/sql/JavaSqlITCase.java @@ -28,7 +28,7 @@ import org.apache.flink.api.java.typeutils.MapTypeInfo; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase; import org.apache.flink.test.operators.util.CollectionDataSets; import org.apache.flink.types.Row; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/table/JavaTableEnvironmentITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/table/JavaTableEnvironmentITCase.java index 1d808a42a965a..9c0ce6fb8e800 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/table/JavaTableEnvironmentITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/batch/table/JavaTableEnvironmentITCase.java @@ -32,7 +32,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.table.calcite.CalciteConfigBuilder; import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase; import org.apache.flink.table.runtime.utils.TableProgramsTestBase; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/FunctionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/FunctionITCase.java index 602fc0404f55e..2ca2135252535 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/FunctionITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/FunctionITCase.java @@ -23,7 +23,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.ValidationException; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.CatalogFunction; import org.apache.flink.table.catalog.ObjectPath; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/JavaSqlITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/JavaSqlITCase.java index dbc9628bb6b97..798d41395617d 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/JavaSqlITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/sql/JavaSqlITCase.java @@ -27,7 +27,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.runtime.utils.JavaStreamTestData; import org.apache.flink.table.runtime.utils.StreamITCase; import org.apache.flink.test.util.AbstractTestBase; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/FunctionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/FunctionITCase.java index c627566fe0b04..509005dbc2602 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/FunctionITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/FunctionITCase.java @@ -24,7 +24,7 @@ import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.ValidationException; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.functions.TableFunction; import org.apache.flink.test.util.AbstractTestBase; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/ValuesITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/ValuesITCase.java index a50074b7dfd46..bf7d0a4dd067c 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/ValuesITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/runtime/stream/table/ValuesITCase.java @@ -24,7 +24,7 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.runtime.utils.StreamITCase; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.utils.TypeConversions; diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala index 57abf503f3c60..a55927fe843f2 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentITCase.scala @@ -25,9 +25,9 @@ import org.apache.flink.core.fs.FileSystem.WriteMode import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment => ScalaStreamExecutionEnvironment} import org.apache.flink.table.api.TableEnvironmentITCase.getPersonCsvTableSource +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.{StreamTableEnvironment => ScalaStreamTableEnvironment} import org.apache.flink.table.api.internal.{TableEnvironmentImpl, TableEnvironmentInternal} -import org.apache.flink.table.api.java.StreamTableEnvironment -import org.apache.flink.table.api.scala.{StreamTableEnvironment => ScalaStreamTableEnvironment} import org.apache.flink.table.runtime.utils.StreamITCase import org.apache.flink.table.sinks.CsvTableSink import org.apache.flink.table.sources.CsvTableSource @@ -35,19 +35,21 @@ import org.apache.flink.table.utils.TableTestUtil.{readFromResource, replaceStag import org.apache.flink.table.utils.{TestTableSourceWithTime, TestingOverwritableTableSink} import org.apache.flink.types.Row import org.apache.flink.util.FileUtils + import org.apache.flink.shaded.guava18.com.google.common.collect.Lists + import org.hamcrest.Matchers.containsString import org.junit.Assert.{assertEquals, assertFalse, assertTrue} import org.junit.rules.{ExpectedException, TemporaryFolder} import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.{After, Before, Rule, Test} -import _root_.java.io.{File, FileOutputStream, OutputStreamWriter} -import _root_.java.lang.{Long => JLong} -import _root_.java.util -import _root_.scala.collection.mutable +import java.io.{File, FileOutputStream, OutputStreamWriter} +import java.lang.{Long => JLong} +import java.util +import scala.collection.mutable @RunWith(classOf[Parameterized]) class TableEnvironmentITCase(tableEnvName: String) { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala index ae6b6afe8b3ac..99cfcec033c4d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableEnvironmentTest.scala @@ -19,17 +19,16 @@ package org.apache.flink.table.api import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.tuple.{Tuple3 => JTuple3} +import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ +import org.apache.flink.api.scala.typeutils.UnitTypeInfo import org.apache.flink.table.api.TableEnvironmentTest._ import org.apache.flink.table.api.Types._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo.{PROCTIME_INDICATOR => PROCTIME} -import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo.{ROWTIME_INDICATOR => ROWTIME} +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo.{PROCTIME_INDICATOR => PROCTIME, ROWTIME_INDICATOR => ROWTIME} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row -import org.apache.flink.api.java.tuple.{Tuple3 => JTuple3} -import org.apache.flink.api.java.typeutils.GenericTypeInfo -import org.apache.flink.api.scala.typeutils.UnitTypeInfo + import org.junit.Assert.assertEquals import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableITCase.scala index b0eb4c7adab77..6849ac08c81d2 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableITCase.scala @@ -20,17 +20,20 @@ package org.apache.flink.table.api import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.table.api.TableEnvironmentITCase.getPersonCsvTableSource +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment import org.apache.flink.table.api.internal.{TableEnvironmentImpl, TableEnvironmentInternal} -import org.apache.flink.table.api.java.StreamTableEnvironment import org.apache.flink.types.Row + import org.apache.flink.shaded.guava18.com.google.common.collect.Lists + import org.hamcrest.Matchers.containsString import org.junit.Assert.{assertEquals, assertTrue} import org.junit.rules.{ExpectedException, TemporaryFolder} import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.{Before, Rule, Test} -import _root_.java.util + +import java.util @RunWith(classOf[Parameterized]) class TableITCase(tableEnvName: String) { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala index 22bb32542d6c3..247a44216065f 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala @@ -18,24 +18,23 @@ package org.apache.flink.table.api -import _root_.java.util.{HashMap => JHashMap} -import _root_.java.util.{Map => JMap} -import _root_.java.sql.{Date, Time, Timestamp} - import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo, TypeInformation} import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.descriptors.{ConnectorDescriptor, Schema} import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR +import org.apache.flink.table.descriptors.{ConnectorDescriptor, Schema} import org.apache.flink.table.expressions.utils._ import org.apache.flink.table.runtime.utils.CommonTestData import org.apache.flink.table.sources.{CsvTableSource, TableSource} import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{TableTestBase, TestFilterableTableSource} import org.apache.flink.types.Row + import org.junit.{Assert, Test} +import java.sql.{Date, Time, Timestamp} +import java.util.{HashMap => JHashMap, Map => JMap} + class TableSourceTest extends TableTestBase { private val projectedFields: Array[String] = Array("last", "id", "score") diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/BatchTableEnvironmentTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/BatchTableEnvironmentTest.scala index 87e0a9f3f7cfd..d56af890edce2 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/BatchTableEnvironmentTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/BatchTableEnvironmentTest.scala @@ -19,15 +19,14 @@ package org.apache.flink.table.api.batch import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{ResultKind, TableException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.catalog.{GenericInMemoryCatalog, ObjectPath} import org.apache.flink.table.runtime.stream.sql.FunctionITCase.{SimpleScalarFunction, TestUDF} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil.{readFromResource, replaceStageId, _} import org.apache.flink.types.Row -import org.hamcrest.Matchers.containsString import org.junit.Assert.{assertEquals, assertFalse, assertTrue, fail} import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala index 34e7581857320..a9f0847579fd4 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/ExplainTest.scala @@ -20,14 +20,15 @@ package org.apache.flink.table.api.batch import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.BatchTableEnvironmentImpl import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.BatchTableEnvironmentImpl -import org.apache.flink.table.api.{Table, Types} import org.apache.flink.table.runtime.utils.CommonTestData import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.table.utils.TableTestUtil.{batchTableNode, readFromResource, replaceStageId} import org.apache.flink.test.util.MultipleProgramsTestBase + import org.junit.Assert.assertEquals import org.junit._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/AggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/AggregateTest.scala index 07440fd49d99f..aadf6851ebdff 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/AggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/AggregateTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test /** diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CalcTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CalcTest.scala index 6b66c0e7b3bd8..8244e9e441a15 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CalcTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CalcTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase +import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class CalcTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CorrelateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CorrelateTest.scala index c776e1b84b9b4..dd6b1751f322b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CorrelateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/CorrelateTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedTableFunctions.JavaVarsArgTableFunc0 import org.apache.flink.table.utils.TableTestUtil._ -import org.apache.flink.table.utils.{HierarchyTableFunction, PojoTableFunc, TableFunc2, _} +import org.apache.flink.table.utils._ + import org.junit.Test class CorrelateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala index ff5e560b5a68c..7b36a4048b1ca 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class DistinctAggregateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala index 74f71c17ecde9..e59ab12ee8ed7 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala @@ -18,15 +18,16 @@ package org.apache.flink.table.api.batch.sql -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test +import java.sql.Timestamp + class GroupWindowTest extends TableTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupingSetsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupingSetsTest.scala index 59faa2c585df3..7e65fdadfcd84 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupingSetsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/GroupingSetsTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class GroupingSetsTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala index 5709b256d9721..2b0f091bf5c91 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class JoinTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SetOperatorsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SetOperatorsTest.scala index 86e5ab8786805..db942fae18165 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SetOperatorsTest.scala @@ -21,8 +21,7 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{Types, _} import org.apache.flink.table.runtime.utils.CommonTestData.NonPojo import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SingleRowJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SingleRowJoinTest.scala index 4142d880e6616..d6e34ba199b74 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SingleRowJoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/SingleRowJoinTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase +import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class SingleRowJoinTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/TemporalTableJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/TemporalTableJoinTest.scala index bed8a5ad961bf..ca94d9a285960 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/TemporalTableJoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/TemporalTableJoinTest.scala @@ -17,16 +17,15 @@ */ package org.apache.flink.table.api.batch.sql -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils._ + import org.hamcrest.Matchers.startsWith import org.junit.Test +import java.sql.Timestamp + class TemporalTableJoinTest extends TableTestBase { val util: TableTestUtil = batchTestUtil() diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CalcValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CalcValidationTest.scala index 69b12b2da0292..2edcffbbb6258 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CalcValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CalcValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.batch.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class CalcValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CorrelateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CorrelateValidationTest.scala index 81381f3629bb8..f92fe2ade5709 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CorrelateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/CorrelateValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.batch.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{TableFunc1, TableTestBase} + import org.junit.Test class CorrelateValidationTest extends TableTestBase{ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/GroupWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/GroupWindowValidationTest.scala index e32b0a94b613c..3dc58cd73a74c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/GroupWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/GroupWindowValidationTest.scala @@ -18,15 +18,15 @@ package org.apache.flink.table.api.batch.sql.validation -import java.sql.Timestamp - import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.utils.TableTestBase + import org.junit.Test +import java.sql.Timestamp + class GroupWindowValidationTest extends TableTestBase { @Test(expected = classOf[TableException]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala index 60208f99d1b78..05a36565b3ae7 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala @@ -20,10 +20,10 @@ package org.apache.flink.table.api.batch.sql.validation import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Types, ValidationException} import org.apache.flink.table.utils.{MemoryTableSourceSinkUtil, TableTestBase} + import org.junit._ class InsertIntoValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/JoinValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/JoinValidationTest.scala index 628bf5f6888bc..20f3258697769 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/JoinValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/JoinValidationTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.api.batch.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row + import org.junit.Test class JoinValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/OverWindowValidationTest.scala index dfbdd5acb808d..50aff68e0ea99 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/OverWindowValidationTest.scala @@ -18,15 +18,15 @@ package org.apache.flink.table.api.batch.sql.validation -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.OverAgg0 -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.TableTestBase + import org.junit.Test +import java.sql.Timestamp + class OverWindowValidationTest extends TableTestBase { /** diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/SortValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/SortValidationTest.scala index cfc80676cef0c..288f7a451e43e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/SortValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/validation/SortValidationTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.api.batch.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row + import org.junit.Test class SortValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/AggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/AggregateTest.scala index 8f458504b1391..2d00687e5d441 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/AggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/AggregateTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase +import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test /** diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CalcTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CalcTest.scala index 42a6c84c6134a..f46cef7897239 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CalcTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CalcTest.scala @@ -20,9 +20,8 @@ package org.apache.flink.table.api.batch.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala.createTypeInformation -import org.apache.flink.table.api.DataTypes +import org.apache.flink.table.api._ import org.apache.flink.table.api.batch.table.CalcTest.{MyHashCode, TestCaseClass, WC, giveMeCaseClass} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/ColumnFunctionsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/ColumnFunctionsTest.scala index f325f0c7779d0..b085bed7f6216 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/ColumnFunctionsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/ColumnFunctionsTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.api.batch.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Table -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{BatchTableTestUtil, TableTestBase} + import org.junit.Test /** diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CorrelateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CorrelateTest.scala index 9ec49d1a13556..dba7270e80df0 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CorrelateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/CorrelateTest.scala @@ -18,15 +18,15 @@ package org.apache.flink.table.api.batch.table -import org.apache.calcite.rel.rules.{CalcMergeRule, FilterCalcMergeRule, ProjectCalcMergeRule} -import org.apache.calcite.tools.RuleSets import org.apache.flink.api.scala._ -import org.apache.flink.table.api.PlannerConfig -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.calcite.CalciteConfigBuilder import org.apache.flink.table.plan.rules.FlinkRuleSets import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{TableFunc0, TableFunc1, TableTestBase} + +import org.apache.calcite.rel.rules.{CalcMergeRule, FilterCalcMergeRule, ProjectCalcMergeRule} +import org.apache.calcite.tools.RuleSets import org.junit.Test import scala.collection.JavaConversions._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala index 51eb170ac3eec..6ab92180dbe43 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala @@ -18,16 +18,16 @@ package org.apache.flink.table.api.batch.table -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test +import java.sql.Timestamp + class GroupWindowTest extends TableTestBase { //=============================================================================================== diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/JoinTest.scala index 1dcb80382ef79..638f4698a6425 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/JoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/JoinTest.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.api.batch.table import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.batch.table.JoinTest.Merger -import org.apache.flink.table.api.scala._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class JoinTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/SetOperatorsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/SetOperatorsTest.scala index b33c6a5bc7e5b..9868efaefb041 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/SetOperatorsTest.scala @@ -21,8 +21,7 @@ package org.apache.flink.table.api.batch.table import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.CommonTestData.NonPojo import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/TemporalTableJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/TemporalTableJoinTest.scala index d8dd873cbb1d2..f33378d7042bb 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/TemporalTableJoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/TemporalTableJoinTest.scala @@ -17,15 +17,15 @@ */ package org.apache.flink.table.api.batch.table -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils._ + import org.hamcrest.Matchers.startsWith import org.junit.Test +import java.sql.Timestamp + class TemporalTableJoinTest extends TableTestBase { val util: TableTestUtil = batchTestUtil() diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/AggregateStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/AggregateStringExpressionTest.scala index 4e7270ffcd743..f0e694d2d90cc 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/AggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/AggregateStringExpressionTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.api.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction +import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset import org.apache.flink.table.utils.TableTestBase + import org.junit._ class AggregateStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CalcStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CalcStringExpressionTest.scala index 7b706f3d01f0f..0a0ba47feb233 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CalcStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CalcStringExpressionTest.scala @@ -18,16 +18,16 @@ package org.apache.flink.table.api.batch.table.stringexpr -import java.sql.{Date, Time, Timestamp} - import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets.CustomType -import org.apache.flink.table.api.Types import org.apache.flink.table.api.Types._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{Types, _} import org.apache.flink.table.utils.TableTestBase + import org.junit._ +import java.sql.{Date, Time, Timestamp} + class CalcStringExpressionTest extends TableTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CorrelateStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CorrelateStringExpressionTest.scala index 6c61fd0d63471..798a3e4d73c24 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CorrelateStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/CorrelateStringExpressionTest.scala @@ -22,8 +22,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.java.{DataSet => JDataSet} import org.apache.flink.api.scala._ import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.{Expressions, Types} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{Types, _} import org.apache.flink.table.utils.{PojoTableFunc, TableFunc2, _} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/JoinStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/JoinStringExpressionTest.scala index 15c35dfc15442..60e234d1c5b8e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/JoinStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/JoinStringExpressionTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.expressions.Literal +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase + import org.junit._ class JoinStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SetOperatorsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SetOperatorsTest.scala index 14213ca261287..15724a0bc8e88 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SetOperatorsTest.scala @@ -18,14 +18,15 @@ package org.apache.flink.table.api.batch.table.stringexpr -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test +import java.sql.Timestamp + class SetOperatorsTest extends TableTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SortStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SortStringExpressionTest.scala index 204ec7726aa83..192291fcfc2bb 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SortStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/stringexpr/SortStringExpressionTest.scala @@ -19,8 +19,9 @@ package org.apache.flink.table.api.batch.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class SortStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/AggregateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/AggregateValidationTest.scala index 21fdea112cb67..d1e47d5b83564 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/AggregateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/AggregateValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.TableTestBase + import org.junit._ class AggregateValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CalcValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CalcValidationTest.scala index 8aa4f34cc82b6..b168aa90f415b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CalcValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CalcValidationTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CorrelateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CorrelateValidationTest.scala index ce22b0eb669d6..663e039f8995b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CorrelateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/CorrelateValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{TableFunc1, TableTestBase} + import org.junit.Test class CorrelateValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/GroupWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/GroupWindowValidationTest.scala index 15d70586010ce..44541bb6ce208 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/GroupWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/GroupWindowValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class GroupWindowValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/InsertIntoValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/InsertIntoValidationTest.scala index 4e39475528d36..579fc7cc84664 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/InsertIntoValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/InsertIntoValidationTest.scala @@ -20,10 +20,10 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.{Types, ValidationException} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.{MemoryTableSourceSinkUtil, TableTestBase} + import org.junit._ class InsertIntoValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/JoinValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/JoinValidationTest.scala index 6d686172a4038..b68d9641cdd80 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/JoinValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/JoinValidationTest.scala @@ -20,10 +20,11 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.{TableException, ValidationException, _} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row + import org.junit._ class JoinValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/OverWindowValidationTest.scala index b19a523675771..ba008335e03cb 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/OverWindowValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Tumble, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.OverAgg0 -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.TableTestBase + import org.junit._ class OverWindowValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SetOperatorsValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SetOperatorsValidationTest.scala index 04243959da6da..275a5e2672100 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SetOperatorsValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SetOperatorsValidationTest.scala @@ -20,9 +20,10 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestBase + import org.junit._ class SetOperatorsValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala index 24156e483b42c..3575833af6038 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/table/validation/SortValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.batch.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase + import org.junit._ class SortValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala index 6f99d9abc4dac..b38ff12a9fada 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/ExplainTest.scala @@ -21,13 +21,14 @@ package org.apache.flink.table.api.stream import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Table, Types} import org.apache.flink.table.runtime.utils.CommonTestData import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.table.utils.TableTestUtil.{readFromResource, replaceStageId, streamTableNode} import org.apache.flink.test.util.AbstractTestBase + import org.junit.Assert.assertEquals import org.junit._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentTest.scala index bb710c62fc348..5ef115852be1c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentTest.scala @@ -26,16 +26,16 @@ import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JStreamExecEnv} import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.java.internal.{StreamTableEnvironmentImpl => JStreamTableEnvironmentImpl} -import org.apache.flink.table.api.java.{StreamTableEnvironment => JStreamTableEnv} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, ResultKind, TableConfig, TableException, Types, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.java.internal.{StreamTableEnvironmentImpl => JStreamTableEnvironmentImpl} +import org.apache.flink.table.api.bridge.java.{StreamTableEnvironment => JStreamTableEnv} +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.catalog.FunctionCatalog import org.apache.flink.table.executor.StreamExecutor import org.apache.flink.table.module.ModuleManager import org.apache.flink.table.planner.StreamPlanner import org.apache.flink.table.runtime.utils.StreamTestData -import org.apache.flink.table.utils.TableTestUtil.{binaryNode, readFromResource, replaceStageId, streamTableNode, term, unaryNode} +import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{CatalogManagerMocks, TableTestBase} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala index e92cc0f06e3ca..c0752a4ce6888 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/StreamTableEnvironmentValidationTest.scala @@ -18,16 +18,17 @@ package org.apache.flink.table.api.stream -import java.math.BigDecimal import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.{EnvironmentSettings, TableException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.stream.TimeAttributesITCase.TimestampWithEqualWatermark import org.apache.flink.table.utils.TableTestBase import org.junit.Test +import java.math.BigDecimal + class StreamTableEnvironmentValidationTest extends TableTestBase { // ---------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/AggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/AggregateTest.scala index da4242ae9cc0c..f7da2d682202f 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/AggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/AggregateTest.scala @@ -23,11 +23,10 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl -import org.apache.flink.table.api.{TableConfig, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.catalog.{FunctionCatalog, UnresolvedIdentifier} -import org.apache.flink.table.delegation.{Executor, Planner} +import org.apache.flink.table.delegation.Executor import org.apache.flink.table.functions.{AggregateFunction, AggregateFunctionDefinition} import org.apache.flink.table.module.ModuleManager import org.apache.flink.table.utils.TableTestUtil.{streamTableNode, term, unaryNode} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/CorrelateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/CorrelateTest.scala index 89ccb763fdac0..ccd42e8578e4d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/CorrelateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/CorrelateTest.scala @@ -19,12 +19,12 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedTableFunctions.JavaVarsArgTableFunc0 import org.apache.flink.table.utils.TableTestUtil._ -import org.apache.flink.table.utils.{HierarchyTableFunction, PojoTableFunc, TableFunc2, _} +import org.apache.flink.table.utils._ import org.apache.flink.types.Row + import org.junit.Test class CorrelateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala index 348c408a71d29..84fed6dcdb275 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} import org.junit.{Ignore, Test} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala index 733470dce6359..e169b1b221d32 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala index dcc86ab8b49a6..39b1d2f94c828 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala @@ -17,15 +17,15 @@ */ package org.apache.flink.table.api.stream.sql -import org.apache.calcite.rel.logical.LogicalJoin import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.calcite.RelTimeIndicatorConverter import org.apache.flink.table.planner.StreamPlanner import org.apache.flink.table.runtime.join.WindowJoinUtil -import org.apache.flink.table.utils.TableTestUtil.{term, _} +import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + +import org.apache.calcite.rel.logical.LogicalJoin import org.junit.Assert._ import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/MatchRecognizeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/MatchRecognizeTest.scala index 9088bd0da21c9..7a2f412dfe994 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/MatchRecognizeTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/MatchRecognizeTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestUtil.{term, _} import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/OverWindowTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/OverWindowTest.scala index dfa695b0fbfb7..130ce5ad2a65d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/OverWindowTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/OverWindowTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SetOperatorsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SetOperatorsTest.scala index e71c45e58fdc9..9b51267dc1bd0 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SetOperatorsTest.scala @@ -18,7 +18,8 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.TableTestBase import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SortTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SortTest.scala index b8ad376c7e7d7..8595bddb0e97d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SortTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/SortTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala index cad41e9ed4b4e..7345c5cc67eaa 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/TemporalTableJoinTest.scala @@ -17,17 +17,17 @@ */ package org.apache.flink.table.api.stream.sql -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.plan.logical.rel.LogicalTemporalTableJoin.TEMPORAL_JOIN_CONDITION import org.apache.flink.table.utils.TableTestUtil.{binaryNode, streamTableNode, term, unaryNode} import org.apache.flink.table.utils._ + import org.hamcrest.Matchers.startsWith import org.junit.Test +import java.sql.Timestamp + class TemporalTableJoinTest extends TableTestBase { val util: TableTestUtil = streamTestUtil() diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/UnionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/UnionTest.scala index 9a039d98c8a92..bbaeff733b019 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/UnionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/UnionTest.scala @@ -21,11 +21,10 @@ package org.apache.flink.table.api.stream.sql import org.apache.flink.api.java.typeutils.GenericTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.{Expressions, Types} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.CommonTestData.NonPojo -import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.TableTestBase +import org.apache.flink.table.utils.TableTestUtil._ import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/CorrelateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/CorrelateValidationTest.scala index 926236e7b9c2e..2693ae6b15a9a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/CorrelateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/CorrelateValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{TableFunc1, TableTestBase} + import org.junit.Test class CorrelateValidationTest extends TableTestBase{ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala index 293b9fb2b62ac..e7691e965aaa8 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala @@ -21,7 +21,7 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{EnvironmentSettings, Types, ValidationException} import org.apache.flink.table.runtime.utils.StreamTestData import org.apache.flink.table.utils.MemoryTableSourceSinkUtil diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala index bbfd2f4549fc6..8aa63a06bbf4c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/JoinValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.hamcrest.Matchers import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala index 4048e52f3d202..c8c3a2dc568fc 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/MatchRecognizeValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.hamcrest.Matchers import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala index d04b6d007bd4a..6776bba7fe619 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/OverWindowValidationTest.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.OverAgg0 -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row + import org.junit.Test class OverWindowValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/SortValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/SortValidationTest.scala index 6c477fd9ca9f5..70d99d65f5c80 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/SortValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/SortValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.TableException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.junit.Test class SortValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala index 5c237ffc5ee95..46252512552c5 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/validation/WindowAggregateValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.stream.sql.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.junit.Test class WindowAggregateValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala index 1250f0d512dc7..6ddf133c1adf2 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala @@ -20,11 +20,11 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg -import org.apache.flink.table.utils.{CountMinMax, TableTestBase} import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.utils.{CountMinMax, TableTestBase} + import org.junit.Test class AggregateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CalcTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CalcTest.scala index 0ee826ac73634..077b3f789750f 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CalcTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CalcTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.Tumble import org.apache.flink.table.expressions.utils.{Func1, Func23, Func24} import org.apache.flink.table.utils.TableTestBase diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/ColumnFunctionsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/ColumnFunctionsTest.scala index b476790c80c36..4c4e8be412cee 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/ColumnFunctionsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/ColumnFunctionsTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{Over, Slide, Table} import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.functions.aggfunctions.CountAggFunction diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CorrelateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CorrelateTest.scala index 80b8d6f68cdf4..d2ba3e3ded0aa 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CorrelateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/CorrelateTest.scala @@ -17,16 +17,16 @@ */ package org.apache.flink.table.api.stream.table -import org.apache.calcite.rel.rules.{CalcMergeRule, FilterCalcMergeRule, ProjectCalcMergeRule} -import org.apache.calcite.tools.RuleSets import org.apache.flink.api.scala._ -import org.apache.flink.table.api.PlannerConfig -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.calcite.CalciteConfigBuilder import org.apache.flink.table.expressions.utils.Func13 import org.apache.flink.table.plan.rules.FlinkRuleSets import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils._ + +import org.apache.calcite.rel.rules.{CalcMergeRule, FilterCalcMergeRule, ProjectCalcMergeRule} +import org.apache.calcite.tools.RuleSets import org.junit.Test import scala.collection.JavaConversions._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTableAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTableAggregateTest.scala index db869aede8391..4a239bfbe784a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTableAggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTableAggregateTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.utils.{EmptyTableAggFunc, EmptyTableAggFuncWithIntResultType, TableTestBase} +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.utils.{EmptyTableAggFunc, EmptyTableAggFuncWithIntResultType, TableTestBase} + import org.junit.Test class GroupWindowTableAggregateTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala index 6fb46d597e800..c5eb3632ec397 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.{Ignore, Test} class GroupWindowTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala index 0b9dec55dcd4b..adc6417bef739 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/JoinTest.scala @@ -21,7 +21,8 @@ package org.apache.flink.table.api.stream.table import java.sql.Timestamp import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/OverWindowTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/OverWindowTest.scala index 7214ae962277f..0be16ee1f1f57 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/OverWindowTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/OverWindowTest.scala @@ -18,12 +18,12 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithRetract -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.Func1 -import org.apache.flink.table.api.{Over, Table} +import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithRetract import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.junit.Test class OverWindowTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/SetOperatorsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/SetOperatorsTest.scala index a09e0bb68fec1..ace5261575653 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/SetOperatorsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/SetOperatorsTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil.{binaryNode, streamTableNode, term, unaryNode} import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableAggregateTest.scala index a8ac82c1dc285..c230031e53df5 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableAggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableAggregateTest.scala @@ -21,11 +21,10 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api.Expressions.$ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{Types, _} import org.apache.flink.table.expressions.utils.Func0 -import org.apache.flink.table.utils.{EmptyTableAggFunc, EmptyTableAggFuncWithIntResultType, TableTestBase} import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.utils.{EmptyTableAggFunc, EmptyTableAggFuncWithIntResultType, TableTestBase} import org.apache.flink.types.Row import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableSourceTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableSourceTest.scala index 8ea3b6703dd89..8c2d42646aa8a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableSourceTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TableSourceTest.scala @@ -20,12 +20,12 @@ package org.apache.flink.table.api.stream.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.{Over, TableSchema, Tumble, Types} -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{TableTestBase, TestNestedProjectableTableSource, TestProjectableTableSource, TestTableSourceWithTime} import org.apache.flink.types.Row + import org.junit.Test class TableSourceTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala index 705355ef3b1de..1b1d0885d6fab 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala @@ -21,7 +21,8 @@ package org.apache.flink.table.api.stream.table import java.sql.Timestamp import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{DataTypes, TableSchema, ValidationException} import org.apache.flink.table.expressions.{Expression, FieldReferenceExpression} import org.apache.flink.table.functions.{TemporalTableFunction, TemporalTableFunctionImpl} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala index 7de35f304c978..bb54c40166bdc 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/AggregateStringExpressionTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Tumble -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMergeAndReset} import org.apache.flink.table.utils.{CountMinMax, TableTestBase} + import org.junit.Test class AggregateStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CalcStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CalcStringExpressionTest.scala index b5f4298ab2678..066c981a8ecee 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CalcStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CalcStringExpressionTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.Func23 import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class CalcStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CorrelateStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CorrelateStringExpressionTest.scala index 80cd8c443d39b..413fe3c32acbc 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CorrelateStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/CorrelateStringExpressionTest.scala @@ -21,7 +21,7 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.utils._ import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowStringExpressionTest.scala index b1e5860b25a2f..5e65251809151 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowStringExpressionTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg +import org.apache.flink.table.api._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction +import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class GroupWindowStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala index a4e5c543e8719..19fa07bba36e5 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/GroupWindowTableAggregateStringExpressionTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{TableTestBase, Top3} + import org.junit.Test class GroupWindowTableAggregateStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/OverWindowStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/OverWindowStringExpressionTest.scala index f3b312bdf7e5d..34c2d6924cc08 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/OverWindowStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/OverWindowStringExpressionTest.scala @@ -19,12 +19,11 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Over -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithRetract -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.Func1 +import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithRetract} import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class OverWindowStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala index bfb3ef63355a1..b72606bb55e84 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/SetOperatorsStringExpressionTest.scala @@ -19,8 +19,9 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class SetOperatorsStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/TableAggregateStringExpressionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/TableAggregateStringExpressionTest.scala index 4119faefef67a..601f1bacc96ed 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/TableAggregateStringExpressionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/stringexpr/TableAggregateStringExpressionTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.api.stream.table.stringexpr import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.Func0 import org.apache.flink.table.utils.{TableTestBase, Top3WithMapView} + import org.junit.Test class TableAggregateStringExpressionTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/AggregateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/AggregateValidationTest.scala index 344bd1c2a0ed8..d6f4097a69622 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/AggregateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/AggregateValidationTest.scala @@ -19,9 +19,9 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{ExpressionParserException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{CountMinMax, TableFunc0, TableTestBase} + import org.junit.Test class AggregateValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CalcValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CalcValidationTest.scala index b5bf19499acde..28313a7619d6b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CalcValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CalcValidationTest.scala @@ -17,15 +17,15 @@ */ package org.apache.flink.table.api.stream.table.validation -import java.math.BigDecimal - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableException, Tumble, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.utils.{TableFunc0, TableTestBase} + import org.junit.Test +import java.math.BigDecimal + class CalcValidationTest extends TableTestBase { @Test(expected = classOf[ValidationException]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala index 74253b65b8a39..c0c0220502ef4 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/CorrelateValidationTest.scala @@ -19,7 +19,7 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.utils.{ObjectTableFunction, TableFunc1, TableFunc2, TableTestBase} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowTableAggregateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowTableAggregateValidationTest.scala index 714e1dd57f6f0..5f290b22741a2 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowTableAggregateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowTableAggregateValidationTest.scala @@ -18,10 +18,10 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge import org.apache.flink.table.utils.{TableTestBase, Top3} + import org.junit.Test class GroupWindowTableAggregateValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowValidationTest.scala index 2b03bc6bc3d3e..1b2427023038a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/GroupWindowValidationTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMerge -import org.apache.flink.table.api.scala._ import org.apache.flink.table.utils.{CountMinMax, TableTestBase} + import org.junit.Test class GroupWindowValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/InsertIntoValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/InsertIntoValidationTest.scala index 75f708364e76c..8db53117e1182 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/InsertIntoValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/InsertIntoValidationTest.scala @@ -20,11 +20,12 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Types, ValidationException} import org.apache.flink.table.runtime.utils.StreamTestData import org.apache.flink.table.utils.MemoryTableSourceSinkUtil + import org.junit.Test class InsertIntoValidationTest { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/JoinValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/JoinValidationTest.scala index 578b0e3abdbe9..02210187acc6e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/JoinValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/JoinValidationTest.scala @@ -20,9 +20,9 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.stream.table.validation.JoinValidationTest.WithoutEqualsHashCode -import org.apache.flink.table.api.{EnvironmentSettings, TableException, ValidationException} import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.runtime.utils.StreamTestData import org.apache.flink.table.utils.TableTestBase diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/OverWindowValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/OverWindowValidationTest.scala index 7626aa91a1b87..4a568a4a66652 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/OverWindowValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/OverWindowValidationTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Over, Table, Tumble, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.planner.StreamPlanner import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{OverAgg0, WeightedAvgWithRetract} import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.junit.Test class OverWindowValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/SetOperatorsValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/SetOperatorsValidationTest.scala index e7409453276ea..9f9f45a88b2e9 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/SetOperatorsValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/SetOperatorsValidationTest.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableAggregateValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableAggregateValidationTest.scala index 9e5feb22ba159..d3dce9b0fcd32 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableAggregateValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableAggregateValidationTest.scala @@ -17,14 +17,14 @@ */ package org.apache.flink.table.api.stream.table.validation -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ import org.apache.flink.table.utils.{EmptyTableAggFunc, TableTestBase} + import org.junit.Test +import java.sql.Timestamp + class TableAggregateValidationTest extends TableTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSinkValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSinkValidationTest.scala index 5d1355eb320c6..eb2a2deb22abb 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSinkValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSinkValidationTest.scala @@ -20,12 +20,13 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.TableException +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.runtime.stream.table.{TestAppendSink, TestUpsertSink} import org.apache.flink.table.runtime.utils.StreamTestData import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class TableSinkValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala index 31baa1d6c0655..3c28592f1dd9e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala @@ -20,10 +20,10 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.table.api._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, TableSchema, Types} import org.apache.flink.table.utils.{TableTestBase, TestFilterableTableSourceWithoutExplainSourceOverride, TestProjectableTableSourceWithoutExplainSourceOverride} + import org.hamcrest.Matchers import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TemporalTableJoinValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TemporalTableJoinValidationTest.scala index f62919aff88e4..445139cfa6b89 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TemporalTableJoinValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TemporalTableJoinValidationTest.scala @@ -18,14 +18,14 @@ package org.apache.flink.table.api.stream.table.validation -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils._ + import org.junit.Test +import java.sql.Timestamp + class TemporalTableJoinValidationTest extends TableTestBase { val util: TableTestUtil = streamTestUtil() diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala index 1fe47f491c3bc..73d8e7734ce52 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/UnsupportedOpsValidationTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.api.stream.table.validation import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, ValidationException} +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.{EnvironmentSettings, ValidationException, _} import org.apache.flink.table.runtime.utils.StreamTestData import org.apache.flink.test.util.AbstractTestBase diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/ColumnFunctionsValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/ColumnFunctionsValidationTest.scala index 5443afbc490cf..d7b96ab00ab98 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/ColumnFunctionsValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/ColumnFunctionsValidationTest.scala @@ -18,10 +18,10 @@ package org.apache.flink.table.api.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Slide, ValidationException} +import org.apache.flink.table.api.{Slide, ValidationException, _} import org.apache.flink.table.functions.BuiltInFunctionDefinitions import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + import org.junit.Test /** diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/InlineTableValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/InlineTableValidationTest.scala index a28c0f6241426..4fbc3f40192c6 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/InlineTableValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/InlineTableValidationTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.api.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class InlineTableValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableEnvironmentValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableEnvironmentValidationTest.scala index 527c53a745e32..8cd6953eb1340 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableEnvironmentValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableEnvironmentValidationTest.scala @@ -24,8 +24,8 @@ import org.apache.flink.api.java.typeutils.{GenericTypeInfo, RowTypeInfo, TupleT import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets import org.apache.flink.table.api.TableEnvironmentTest.{CClass, PojoClass} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.{TableException, ValidationException, _} import org.apache.flink.table.runtime.types.CRowTypeInfo import org.apache.flink.table.utils.TableTestBase import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSinksValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSinksValidationTest.scala index e72811e0bdb2a..24aa62d13e975 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSinksValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSinksValidationTest.scala @@ -21,11 +21,11 @@ package org.apache.flink.table.api.validation import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, Types, ValidationException} +import org.apache.flink.table.api.{TableException, Types, ValidationException, _} import org.apache.flink.table.runtime.stream.table.TestAppendSink import org.apache.flink.table.utils.MemoryTableSourceSinkUtil.UnsafeMemoryAppendTableSink import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class TableSinksValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala index b7a816a22c07d..3c5a718708495 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala @@ -22,19 +22,20 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.api.{EnvironmentSettings, TableSchema, Types, ValidationException} import org.apache.flink.table.sources._ import org.apache.flink.table.sources.tsextractors.ExistingField import org.apache.flink.table.sources.wmstrategies.AscendingTimestamps import org.apache.flink.table.utils.{TableTestBase, TestTableSourceWithTime} import org.apache.flink.types.Row + import org.junit.Test + import java.util import java.util.Collections -import org.apache.flink.table.api.internal.TableEnvironmentInternal - class TableSourceValidationTest extends TableTestBase{ val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala index 4ba2d5227f550..a0f2f10c82d03 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala @@ -18,11 +18,11 @@ package org.apache.flink.table.api.validation import org.apache.flink.api.scala._ -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.Func0 import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.OverAgg0 import org.apache.flink.table.utils.TableTestBase + import org.junit.Test class UserDefinedFunctionValidationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/catalog/CatalogTableITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/catalog/CatalogTableITCase.scala index b9f50c1d32812..23a6cd9992a25 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/catalog/CatalogTableITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/catalog/CatalogTableITCase.scala @@ -20,7 +20,7 @@ package org.apache.flink.table.catalog import org.apache.flink.api.scala.ExecutionEnvironment import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala.{BatchTableEnvironment, StreamTableEnvironment} +import org.apache.flink.table.api.bridge.scala.{BatchTableEnvironment, StreamTableEnvironment} import org.apache.flink.table.api.{EnvironmentSettings, TableEnvironment, ValidationException} import org.apache.flink.table.factories.utils.TestCollectionTableFactory import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala index 1bb1543123893..4e41a5131a184 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ArrayTypeTest.scala @@ -18,13 +18,13 @@ package org.apache.flink.table.expressions -import java.sql.Date - -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ArrayTypeTestBase + import org.junit.Test +import java.sql.Date + class ArrayTypeTest extends ArrayTypeTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala index c27abb85b4318..8f1e5788c6150 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/CompositeAccessTest.scala @@ -18,8 +18,9 @@ package org.apache.flink.table.expressions -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.CompositeTypeTestBase + import org.junit.Test class CompositeAccessTest extends CompositeTypeTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DateTimeFunctionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DateTimeFunctionTest.scala index a04354313c141..6e3114359c0de 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DateTimeFunctionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DateTimeFunctionTest.scala @@ -18,16 +18,17 @@ package org.apache.flink.table.expressions -import java.sql.Timestamp - import org.apache.flink.api.common.typeinfo.{TypeInformation, Types} import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ExpressionTestBase import org.apache.flink.types.Row + import org.joda.time.{DateTime, DateTimeZone} import org.junit.Test +import java.sql.Timestamp + class DateTimeFunctionTest extends ExpressionTestBase { private val INSTANT = DateTime.parse("1990-01-02T03:04:05.678Z") private val LOCAL_ZONE = DateTimeZone.getDefault diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala index db8dfd6e1b6df..8473cde00982d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.expressions import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.types.Row import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ExpressionTestBase +import org.apache.flink.types.Row + import org.junit.Test class DecimalTypeTest extends ExpressionTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/LiteralTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/LiteralTest.scala index bf568c7d33d82..8ae347e687da0 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/LiteralTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/LiteralTest.scala @@ -20,11 +20,11 @@ package org.apache.flink.table.expressions import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.{ExpressionTestBase, Func3} import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.types.Row + import org.junit.Test class LiteralTest extends ExpressionTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala index 56cfd0f7cdb12..9cf902cf4e7af 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/MapTypeTest.scala @@ -18,13 +18,13 @@ package org.apache.flink.table.expressions -import java.sql.Date - -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.MapTypeTestBase + import org.junit.Test +import java.sql.Date + class MapTypeTest extends MapTypeTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/NonDeterministicTests.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/NonDeterministicTests.scala index 83f4a65e670d2..3c6dfa47edf52 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/NonDeterministicTests.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/NonDeterministicTests.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.expressions import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.types.Row import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ExpressionTestBase +import org.apache.flink.types.Row + import org.junit.{Ignore, Test} /** diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/RowTypeTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/RowTypeTest.scala index 7893e05d99372..f87347de94a98 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/RowTypeTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/RowTypeTest.scala @@ -18,13 +18,13 @@ package org.apache.flink.table.expressions -import java.sql.Date - -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.RowTypeTestBase + import org.junit.Test +import java.sql.Date + class RowTypeTest extends RowTypeTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala index c1dd520a006e6..7ce8545963fba 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ScalarTypesTestBase + import org.junit.Test class ScalarFunctionsTest extends ScalarTypesTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala index 099d478ee64cd..62d575bb714ec 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/ScalarOperatorsTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.{ScalarOperatorsTestBase, ShouldNotExecuteFunc} + import org.junit.Test class ScalarOperatorsTest extends ScalarOperatorsTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala index a62814060d5eb..5cdf3c0e4b15a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala @@ -18,16 +18,16 @@ package org.apache.flink.table.expressions -import java.sql.{Date, Time, Timestamp} - import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.types.Row -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ExpressionTestBase +import org.apache.flink.types.Row + import org.junit.Test +import java.sql.{Date, Time, Timestamp} + class TemporalTypesTest extends ExpressionTestBase { @Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala index 65423b9b80398..0548eb9f7c94b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/UserDefinedScalarFunctionTest.scala @@ -18,19 +18,19 @@ package org.apache.flink.table.expressions -import java.sql.{Date, Time, Timestamp} - import org.apache.flink.api.common.typeinfo.BasicTypeInfo._ import org.apache.flink.api.common.typeinfo.{BasicArrayTypeInfo, TypeInformation} import org.apache.flink.api.java.typeutils.RowTypeInfo -import org.apache.flink.types.Row -import org.apache.flink.table.api.{Types, ValidationException} -import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.expressions.utils.{ExpressionTestBase, GraduatedStudent, _} +import org.apache.flink.table.api._ +import org.apache.flink.table.expressions.utils._ import org.apache.flink.table.functions.ScalarFunction +import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions._ +import org.apache.flink.types.Row + import org.junit.Test + import java.lang.{Boolean => JBoolean} +import java.sql.{Date, Time, Timestamp} class UserDefinedScalarFunctionTest extends ExpressionTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala index d222e888a54a9..b0b451317319d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/utils/ExpressionTestBase.scala @@ -36,10 +36,10 @@ import org.apache.flink.api.java.{DataSet => JDataSet} import org.apache.flink.api.scala.{DataSet, ExecutionEnvironment} import org.apache.flink.configuration.Configuration import org.apache.flink.core.fs.Path -import org.apache.flink.table.api.scala.BatchTableEnvironment +import org.apache.flink.table.api.bridge.scala.BatchTableEnvironment import org.apache.flink.table.api.TableConfig import org.apache.flink.table.api.internal.TableEnvImpl -import org.apache.flink.table.api.scala.internal.BatchTableEnvironmentImpl +import org.apache.flink.table.api.bridge.scala.internal.BatchTableEnvironmentImpl import org.apache.flink.table.calcite.{CalciteParser, FlinkRelBuilder} import org.apache.flink.table.codegen.{Compiler, FunctionCodeGenerator, GeneratedFunction} import org.apache.flink.table.expressions.{Expression, ExpressionParser} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ArrayTypeValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ArrayTypeValidationTest.scala index d6f70cf18141b..bd8126a9c280b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ArrayTypeValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ArrayTypeValidationTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ArrayTypeTestBase + import org.junit.Test class ArrayTypeValidationTest extends ArrayTypeTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/CompositeAccessValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/CompositeAccessValidationTest.scala index 829fe6568a18a..192c5da69f387 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/CompositeAccessValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/CompositeAccessValidationTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.CompositeTypeTestBase + import org.junit.Test class CompositeAccessValidationTest extends CompositeTypeTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/MapTypeValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/MapTypeValidationTest.scala index 3862466f57dbb..d25e5f9cd0636 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/MapTypeValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/MapTypeValidationTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.MapTypeTestBase + import org.junit.Test class MapTypeValidationTest extends MapTypeTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/RowTypeValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/RowTypeValidationTest.scala index 94e8394a1161d..009452da4c69e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/RowTypeValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/RowTypeValidationTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.RowTypeTestBase + import org.junit.Test class RowTypeValidationTest extends RowTypeTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarFunctionsValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarFunctionsValidationTest.scala index 2d9186bd5821e..26a555a6909e3 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarFunctionsValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarFunctionsValidationTest.scala @@ -18,12 +18,12 @@ package org.apache.flink.table.expressions.validation -import org.apache.calcite.avatica.util.TimeUnit -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{SqlParserException, ValidationException} +import org.apache.flink.table.api._ import org.apache.flink.table.codegen.CodeGenException import org.apache.flink.table.expressions.TimePointUnit import org.apache.flink.table.expressions.utils.ScalarTypesTestBase + +import org.apache.calcite.avatica.util.TimeUnit import org.junit.Test class ScalarFunctionsValidationTest extends ScalarTypesTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarOperatorsValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarOperatorsValidationTest.scala index 2ab3ceffdd48a..1d0cfc508fe09 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarOperatorsValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/expressions/validation/ScalarOperatorsValidationTest.scala @@ -18,9 +18,9 @@ package org.apache.flink.table.expressions.validation -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.ScalarOperatorsTestBase + import org.junit.Test class ScalarOperatorsValidationTest extends ScalarOperatorsTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala index 4e6562da281ff..f2df9ffaddc42 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/MatchRecognizeValidationTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.`match` import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.runtime.stream.sql.ToMillis import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala index a6dd29bcc6484..9c9df9861e7e8 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/match/PatternTranslatorTestBase.scala @@ -23,9 +23,9 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.cep.pattern.Pattern import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream} import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.{EnvironmentSettings, TableConfig} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.operations.QueryOperation import org.apache.flink.table.plan.nodes.datastream.{DataStreamMatch, DataStreamScan} import org.apache.flink.table.planner.StreamPlanner diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/CalcPythonCorrelateTransposeRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/CalcPythonCorrelateTransposeRuleTest.scala index a5bd851be5852..15c93801ae642 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/CalcPythonCorrelateTransposeRuleTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/CalcPythonCorrelateTransposeRuleTest.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.utils.{MockPythonTableFunction, TableTestBase} import org.apache.flink.table.utils.TableTestUtil.{streamTableNode, term, unaryNode} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala index ac9cc6207d9db..825cd203fcc6d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala @@ -19,11 +19,10 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.utils.{Func1, RichFunc1} import org.apache.flink.table.functions.ScalarFunction -import org.apache.flink.table.functions.python.{PythonEnv, PythonFunction, PythonFunctionKind} +import org.apache.flink.table.functions.python.{PythonEnv, PythonFunction} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/NormalizationRulesTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/NormalizationRulesTest.scala index 910b7194bc2a3..2238fddfe9a2d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/NormalizationRulesTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/NormalizationRulesTest.scala @@ -18,14 +18,14 @@ package org.apache.flink.table.plan -import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule -import org.apache.calcite.tools.RuleSets import org.apache.flink.api.scala._ -import org.apache.flink.table.api.PlannerConfig -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.calcite.CalciteConfigBuilder import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + +import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule +import org.apache.calcite.tools.RuleSets import org.junit.Test class NormalizationRulesTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCalcSplitRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCalcSplitRuleTest.scala index e5bb0e0bef718..973fa0954dbb5 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCalcSplitRuleTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCalcSplitRuleTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.{BooleanPandasScalarFunction, BooleanPythonScalarFunction, PandasScalarFunction, PythonScalarFunction} -import org.apache.flink.table.utils.TableTestUtil.{term, _} import org.apache.flink.table.utils.TableTestBase +import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class PythonCalcSplitRuleTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCorrelateSplitRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCorrelateSplitRuleTest.scala index e7dadec97eddc..50e47facc4b2c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCorrelateSplitRuleTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonCorrelateSplitRuleTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.utils.TableTestUtil.{streamTableNode, term, unaryNode} import org.apache.flink.table.utils.{MockPythonTableFunction, TableFunc1, TableTestBase} + import org.junit.Test class PythonCorrelateSplitRuleTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/QueryDecorrelationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/QueryDecorrelationTest.scala index 97eb7c7e49f57..454717ab5aa9d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/QueryDecorrelationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/QueryDecorrelationTest.scala @@ -19,9 +19,10 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class QueryDecorrelationTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/RetractionRulesTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/RetractionRulesTest.scala index e5bab5547ed43..6acd7046b4006 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/RetractionRulesTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/RetractionRulesTest.scala @@ -18,14 +18,14 @@ package org.apache.flink.table.plan -import org.apache.calcite.rel.RelNode import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Table, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.plan.nodes.datastream._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.CountDistinct import org.apache.flink.table.utils.TableTestUtil._ import org.apache.flink.table.utils.{StreamTableTestUtil, TableTestBase} + +import org.apache.calcite.rel.RelNode import org.junit.Assert._ import org.junit.{Ignore, Test} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromCorrelateRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromCorrelateRuleTest.scala index 8918288bce95b..ae51b06f7c07d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromCorrelateRuleTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromCorrelateRuleTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.utils.TableTestUtil.{streamTableNode, term, unaryNode} import org.apache.flink.table.utils.{TableFunc2, TableTestBase} + import org.junit.Test class SplitPythonConditionFromCorrelateRuleTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromJoinRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromJoinRuleTest.scala index 0c806cef8e756..50c22999c1edf 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromJoinRuleTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/SplitPythonConditionFromJoinRuleTest.scala @@ -19,10 +19,11 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.PythonScalarFunction import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test class SplitPythonConditionFromJoinRuleTest extends TableTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala index 7fa3e47094371..854969432fedd 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/TimeIndicatorConversionTest.scala @@ -18,18 +18,18 @@ package org.apache.flink.table.plan -import java.sql.Timestamp - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.Tumble -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ import org.apache.flink.table.expressions.TimeIntervalUnit import org.apache.flink.table.functions.{ScalarFunction, TableFunction} import org.apache.flink.table.plan.TimeIndicatorConversionTest.{ScalarFunc, TableFunc} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ + import org.junit.Test +import java.sql.Timestamp + /** * Tests for [[org.apache.flink.table.calcite.RelTimeIndicatorConverter]]. */ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/UpdatingPlanCheckerTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/UpdatingPlanCheckerTest.scala index c6e859376360f..11cb25570d98d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/UpdatingPlanCheckerTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/UpdatingPlanCheckerTest.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Table, Tumble} +import org.apache.flink.table.api._ import org.apache.flink.table.plan.util.UpdatingPlanChecker import org.apache.flink.table.utils.StreamTableTestUtil + import org.junit.Assert._ import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala index 530727e3d6ff0..b5a852d8fa06a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/AggregateITCase.scala @@ -18,10 +18,10 @@ package org.apache.flink.table.runtime.batch.sql -import org.apache.calcite.runtime.SqlFunctions.{internalToTimestamp => toTimestamp} import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase @@ -30,6 +30,7 @@ import org.apache.flink.table.utils.NonMergableCount import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row +import org.apache.calcite.runtime.SqlFunctions.{internalToTimestamp => toTimestamp} import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/CalcITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/CalcITCase.scala index 5af4586fe313a..3bcaeacf95b4b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/CalcITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/CalcITCase.scala @@ -18,13 +18,10 @@ package org.apache.flink.table.runtime.batch.sql -import java.sql.{Date, Time, Timestamp} -import java.util - import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils.{Func13, SplitUDF} import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.runtime.batch.table.OldHashCode @@ -32,11 +29,15 @@ import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMod import org.apache.flink.table.runtime.utils.{TableProgramsCollectionTestBase, TableProgramsTestBase} import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row -import org.junit._ + import org.junit.Assert.assertEquals +import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.sql.{Date, Time, Timestamp} +import java.util + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/JoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/JoinITCase.scala index 0a6c8681a493b..c4cdbe3403abb 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/JoinITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/JoinITCase.scala @@ -18,22 +18,23 @@ package org.apache.flink.table.runtime.batch.sql -import java.util - import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.Types -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.util + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/PartitionableSinkITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/PartitionableSinkITCase.scala index 409d23ad46abb..3b114b8703fef 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/PartitionableSinkITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/PartitionableSinkITCase.scala @@ -32,7 +32,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.scala.ExecutionEnvironment import org.apache.flink.configuration.Configuration import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala.BatchTableEnvironment +import org.apache.flink.table.api.bridge.scala.BatchTableEnvironment import org.apache.flink.table.api.{DataTypes, TableSchema} import org.apache.flink.table.factories.utils.TestCollectionTableFactory.TestCollectionInputFormat import org.apache.flink.table.runtime.batch.sql.PartitionableSinkITCase._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SetOperatorsITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SetOperatorsITCase.scala index 373ad716df042..8a7af7a0c71f4 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SetOperatorsITCase.scala @@ -20,11 +20,13 @@ package org.apache.flink.table.runtime.batch.sql import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SortITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SortITCase.scala index e1c100be3fa8a..045abd5bccf4c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SortITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/SortITCase.scala @@ -18,15 +18,16 @@ package org.apache.flink.table.runtime.batch.sql +import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.api.scala.{ExecutionEnvironment, _} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.SortTestUtils._ import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala index 4964a6fbe85ee..c68a7c74ee43d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala @@ -24,8 +24,9 @@ import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets import org.apache.flink.core.fs.FileSystem import org.apache.flink.core.fs.FileSystem.WriteMode -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, ResultKind, TableEnvironment, TableEnvironmentITCase, TableResult, TableSchema} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.table.sinks.CsvTableSink @@ -34,18 +35,19 @@ import org.apache.flink.table.utils.{MemoryTableSourceSinkUtil, TestingOverwrita import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row import org.apache.flink.util.FileUtils + import org.apache.flink.shaded.guava18.com.google.common.collect.Lists + import org.junit.Assert.{assertEquals, assertFalse, assertTrue, fail} import org.junit._ import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import org.junit.runners.Parameterized + import java.io.File import java.lang.{Long => JLong} import java.util -import org.apache.flink.table.api.internal.TableEnvironmentInternal - import scala.collection.JavaConverters._ import scala.io.Source diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableSourceITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableSourceITCase.scala index b66013c4432d0..1edc12e38c787 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableSourceITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableSourceITCase.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.runtime.batch.sql import org.apache.flink.api.scala.ExecutionEnvironment +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.runtime.utils.{CommonTestData, TableProgramsCollectionTestBase} import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode +import org.apache.flink.table.runtime.utils.{CommonTestData, TableProgramsCollectionTestBase} import org.apache.flink.test.util.TestBaseUtils + import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala index 9f1d07a85f208..152bd7b63213c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala @@ -18,23 +18,24 @@ package org.apache.flink.table.runtime.batch.table -import java.math.BigDecimal - import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.Types -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinctWithMergeAndReset, WeightedAvgWithMergeAndReset} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction +import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinctWithMergeAndReset, WeightedAvgWithMergeAndReset} import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.table.utils.{NonMergableCount, Top10} import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.math.BigDecimal + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala index a2dbead63ef29..5920c37e59762 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala @@ -18,14 +18,11 @@ package org.apache.flink.table.runtime.batch.table -import java.math.MathContext -import java.sql.{Date, Time, Timestamp} -import java.util - import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets import org.apache.flink.table.api.Types._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode @@ -33,11 +30,16 @@ import org.apache.flink.table.runtime.utils.{TableProgramsCollectionTestBase, Ta import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.test.util.TestBaseUtils.compareResultAsText import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.math.MathContext +import java.sql.{Date, Time, Timestamp} +import java.util + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala index 9a48962897448..24f43ec7b8bdb 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala @@ -18,12 +18,10 @@ package org.apache.flink.table.runtime.batch.table -import java.sql.{Date, Timestamp} - import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{Types, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils.{Func1, Func18, Func20, RichFunc2} import org.apache.flink.table.runtime.utils.JavaUserDefinedTableFunctions.JavaTableFunc0 import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode @@ -32,11 +30,14 @@ import org.apache.flink.table.utils._ import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.sql.{Date, Timestamp} + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/GroupWindowITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/GroupWindowITCase.scala index 80886bc404988..3aed82f7d7775 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/GroupWindowITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/GroupWindowITCase.scala @@ -18,20 +18,21 @@ package org.apache.flink.table.runtime.batch.table -import java.math.BigDecimal - import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{Session, Slide, Tumble} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.math.BigDecimal + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala index 2193182b1f4b6..09e709cd7fd16 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/JoinITCase.scala @@ -18,13 +18,11 @@ package org.apache.flink.table.runtime.batch.table -import java.lang.Iterable - import org.apache.flink.api.common.functions.MapPartitionFunction import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.expressions.Literal +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils.Func20 import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode @@ -33,10 +31,13 @@ import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row import org.apache.flink.util.Collector + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.lang.{Iterable => JIterable} + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) @@ -483,7 +484,7 @@ class JoinITCase( new MapPartitionFunction[(Int, Long, String), (Integer, Long, String)] { override def mapPartition( - vals: Iterable[(Int, Long, String)], + vals: JIterable[(Int, Long, String)], out: Collector[(Integer, Long, String)]): Unit = { val it = vals.iterator() while (it.hasNext) { @@ -501,7 +502,7 @@ class JoinITCase( new MapPartitionFunction[(Int, Long, Int, String, Long), (Integer, Long, Int, String, Long)] { override def mapPartition( - vals: Iterable[(Int, Long, Int, String, Long)], + vals: JIterable[(Int, Long, Int, String, Long)], out: Collector[(Integer, Long, Int, String, Long)]): Unit = { val it = vals.iterator() while (it.hasNext) { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SetOperatorsITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SetOperatorsITCase.scala index 8218169d6448a..4705ce6f5309e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SetOperatorsITCase.scala @@ -20,11 +20,13 @@ package org.apache.flink.table.runtime.batch.table import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala index 29ac5eb93de02..d654872f86a2a 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/SortITCase.scala @@ -18,15 +18,17 @@ package org.apache.flink.table.runtime.batch.table +import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.api.scala.{ExecutionEnvironment, _} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.SortTestUtils._ import org.apache.flink.table.runtime.utils.TableProgramsClusterTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala index f7861cc1ace38..0477b0f49c78d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala @@ -18,22 +18,24 @@ package org.apache.flink.table.runtime.batch.table -import java.util - import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.table.runtime.utils.{TableProgramsCollectionTestBase, TableProgramsTestBase} import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit._ import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.util + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableITCase.scala index 40d14081186a3..636f6ce6155de 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.runtime.batch.table import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{DataTypes, ResultKind, TableSchema} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.types.Row import org.apache.flink.shaded.guava18.com.google.common.collect.Lists diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSinkITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSinkITCase.scala index f198799d16d3b..93020ef421812 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSinkITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSinkITCase.scala @@ -18,24 +18,25 @@ package org.apache.flink.table.runtime.batch.table -import java.io.File - import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.scala._ import org.apache.flink.api.scala.util.CollectionDataSets -import org.apache.flink.api.scala.{ExecutionEnvironment, _} -import org.apache.flink.table.api.Types +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.table.sinks.CsvTableSink import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.table.utils.MemoryTableSourceSinkUtil.UnsafeMemoryOutputFormatTableSink import org.apache.flink.test.util.TestBaseUtils + import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.io.File + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSourceITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSourceITCase.scala index c43900bfebec8..9c2aaa6ce152b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSourceITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/batch/table/TableSourceITCase.scala @@ -18,27 +18,28 @@ package org.apache.flink.table.runtime.batch.table -import java.lang.{Boolean => JBool, Integer => JInt, Long => JLong} -import java.sql.{Date, Time, Timestamp} - -import org.apache.calcite.runtime.SqlFunctions.{internalToTimestamp => toTimestamp} import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo, TypeInformation} import org.apache.flink.api.java.typeutils.{GenericTypeInfo, RowTypeInfo} import org.apache.flink.api.java.{DataSet, ExecutionEnvironment => JExecEnv} import org.apache.flink.api.scala.ExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{TableException, TableSchema, Tumble, Types} import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode import org.apache.flink.table.runtime.utils.{CommonTestData, TableProgramsCollectionTestBase} import org.apache.flink.table.sources.BatchTableSource import org.apache.flink.table.utils._ import org.apache.flink.test.util.TestBaseUtils import org.apache.flink.types.Row + +import org.apache.calcite.runtime.SqlFunctions.{internalToTimestamp => toTimestamp} import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.lang.{Boolean => JBool, Integer => JInt, Long => JLong} +import java.sql.{Date, Time, Timestamp} + import scala.collection.JavaConverters._ @RunWith(classOf[Parameterized]) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala index 3724a4771fe5b..bec4e0a2a3056 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/AggFunctionHarnessTest.scala @@ -18,14 +18,12 @@ package org.apache.flink.table.runtime.harness -import java.lang.{Integer => JInt} -import java.util.concurrent.ConcurrentLinkedQueue import org.apache.flink.api.scala._ import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.runtime.streamrecord.StreamRecord -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.dataview.MapView import org.apache.flink.table.dataview.StateMapView import org.apache.flink.table.runtime.aggregate.GroupAggProcessFunction @@ -35,6 +33,9 @@ import org.apache.flink.types.Row import org.junit.Assert.assertTrue import org.junit.Test +import java.lang.{Integer => JInt} +import java.util.concurrent.ConcurrentLinkedQueue + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala index e659064301549..222920227e61e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/GroupAggregateHarnessTest.scala @@ -23,9 +23,9 @@ import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.operators.KeyedProcessOperator import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.runtime.streamrecord.StreamRecord -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.runtime.aggregate._ import org.apache.flink.table.runtime.harness.HarnessTestBase._ import org.apache.flink.table.runtime.types.CRow diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/MatchHarnessTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/MatchHarnessTest.scala index 8520be473853b..354f25407ca12 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/MatchHarnessTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/MatchHarnessTest.scala @@ -18,18 +18,19 @@ package org.apache.flink.table.runtime.harness -import java.time.{Instant, ZoneId} -import java.util.concurrent.ConcurrentLinkedQueue import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.runtime.streamrecord.StreamRecord -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.types.CRow import org.apache.flink.types.Row import org.junit.Test +import java.time.{Instant, ZoneId} +import java.util.concurrent.ConcurrentLinkedQueue + import scala.collection.mutable class MatchHarnessTest extends HarnessTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TableAggregateHarnessTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TableAggregateHarnessTest.scala index 844d0f69218b0..7ad8447cd7323 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TableAggregateHarnessTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TableAggregateHarnessTest.scala @@ -21,9 +21,9 @@ import org.apache.flink.api.common.time.Time import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.runtime.streamrecord.StreamRecord -import org.apache.flink.table.api.{EnvironmentSettings, TableConfig} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.StreamTableEnvironmentImpl +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.StreamTableEnvironmentImpl import org.apache.flink.table.runtime.types.CRow import org.apache.flink.table.utils.{Top3WithEmitRetractValue, Top3WithMapView} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala index 07871f6fd60e4..c94ec42c8db30 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala @@ -25,8 +25,9 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, TableSchema, Tumble, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.expressions.{ExpressionParser, TimeIntervalUnit} import org.apache.flink.table.plan.TimeIndicatorConversionTest.TableFunc import org.apache.flink.table.runtime.stream.TimeAttributesITCase.{AtomicTimestampWithEqualWatermark, TestPojo, TimestampWithEqualWatermark, TimestampWithEqualWatermarkPojo} @@ -35,14 +36,14 @@ import org.apache.flink.table.runtime.utils.StreamITCase import org.apache.flink.table.utils.{MemoryTableSourceSinkUtil, TestTableSourceWithTime} import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit.{Before, Test} + import java.lang.{Integer => JInt, Long => JLong} import java.math.BigDecimal import java.sql.Timestamp -import org.apache.flink.table.api.internal.TableEnvironmentInternal - import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/InsertIntoITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/InsertIntoITCase.scala index a820f9207a74d..35dcf6ac70040 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/InsertIntoITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/InsertIntoITCase.scala @@ -21,13 +21,14 @@ package org.apache.flink.table.runtime.stream.sql import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Types} import org.apache.flink.table.runtime.stream.table.{RowCollector, TestRetractSink, TestUpsertSink} import org.apache.flink.table.runtime.utils.{StreamTestData, StreamingWithStateTestBase} import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.test.util.TestBaseUtils + import org.junit.Assert._ import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/JoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/JoinITCase.scala index 395c5f2d3ab85..83ffbffd5c0a1 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/JoinITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/JoinITCase.scala @@ -18,20 +18,21 @@ package org.apache.flink.table.runtime.stream.sql -import java.util import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.table.api.{EnvironmentSettings, Types} -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase} import org.apache.flink.types.Row import org.junit.Assert.assertEquals import org.junit._ +import java.util + import scala.collection.mutable class JoinITCase extends StreamingWithStateTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala index 607c2acf685f1..a3fe65605be9b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/MatchRecognizeITCase.scala @@ -18,15 +18,13 @@ package org.apache.flink.table.runtime.stream.sql -import java.sql.Timestamp -import java.util.TimeZone import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.{AggregateFunction, FunctionContext, ScalarFunction} import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction @@ -36,6 +34,9 @@ import org.apache.flink.types.Row import org.junit.Assert.assertEquals import org.junit.{Before, Test} +import java.sql.Timestamp +import java.util.TimeZone + import scala.collection.mutable class MatchRecognizeITCase extends StreamingWithStateTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/OverWindowITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/OverWindowITCase.scala index d7a7cd0d06cbb..1b56d1a4c1f6c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/OverWindowITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/OverWindowITCase.scala @@ -25,8 +25,8 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.AggregateFunction import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.MultiArgCount import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SetOperatorsITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SetOperatorsITCase.scala index aa13ccff85025..a39872929667e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SetOperatorsITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.runtime.stream.sql import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SortITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SortITCase.scala index 1e038b689371f..4c1d77286d467 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SortITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SortITCase.scala @@ -23,14 +23,15 @@ import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.sink.RichSinkFunction import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.{EnvironmentSettings, Types} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction import org.apache.flink.table.runtime.stream.sql.SortITCase.StringRowSelectorSink +import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase} import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala index 318e528601ba9..8e9f100578d3e 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala @@ -25,9 +25,9 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.watermark.Watermark +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Types} import org.apache.flink.table.descriptors.{Rowtime, Schema} import org.apache.flink.table.expressions.utils.Func15 import org.apache.flink.table.runtime.stream.sql.SqlITCase.TimestampAndWatermarkWithOffset @@ -36,6 +36,7 @@ import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction import org.apache.flink.table.runtime.utils.{JavaUserDefinedTableFunctions, StreamITCase, StreamTestData, StreamingWithStateTestBase} import org.apache.flink.table.utils.{InMemoryTableFactory, MemoryTableSourceSinkUtil} import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit._ diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TableSourceITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TableSourceITCase.scala index 63eff0311942b..c12f274ee8e37 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TableSourceITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TableSourceITCase.scala @@ -21,11 +21,12 @@ package org.apache.flink.table.runtime.stream.sql import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.EnvironmentSettings +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.table.runtime.utils.{CommonTestData, StreamITCase} import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit.Test diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala index 805be0b26f923..b71388a960a4b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/sql/TemporalJoinITCase.scala @@ -18,20 +18,21 @@ package org.apache.flink.table.runtime.stream.sql -import java.sql.Timestamp import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.windowing.time.Time -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase} import org.apache.flink.types.Row import org.junit.Assert.assertEquals import org.junit._ +import java.sql.Timestamp + import scala.collection.mutable class TemporalJoinITCase extends StreamingWithStateTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala index 1abd87f188543..2fec0ba7221fe 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala @@ -21,14 +21,15 @@ package org.apache.flink.table.runtime.stream.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Types} import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, DataViewTestAgg, WeightedAvg} import org.apache.flink.table.runtime.utils.StreamITCase.RetractingSink import org.apache.flink.table.runtime.utils.{JavaUserDefinedAggFunctions, StreamITCase, StreamTestData, StreamingWithStateTestBase} import org.apache.flink.table.utils.CountMinMax import org.apache.flink.types.Row + import org.junit.Assert.assertEquals import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CalcITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CalcITCase.scala index a88af95953f9d..5596d2f2733a9 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CalcITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CalcITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.runtime.stream.table import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils._ import org.apache.flink.table.functions.ScalarFunction import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, UserDefinedFunctionTestUtils} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala index 22dce76693dcf..88ab9bdd8a2b0 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala @@ -17,11 +17,10 @@ */ package org.apache.flink.table.runtime.stream.table -import java.lang.{Boolean => JBoolean} import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Types, ValidationException} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.expressions.utils.{Func18, Func20, RichFunc2} import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, _} import org.apache.flink.table.utils._ @@ -31,6 +30,8 @@ import org.apache.flink.types.Row import org.junit.Assert._ import org.junit.{Before, Test} +import java.lang.{Boolean => JBoolean} + import scala.collection.mutable class CorrelateITCase extends AbstractTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala index 0c7e7cc71231e..4c5dd21fa772c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala @@ -23,8 +23,8 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Session, Slide, Tumble} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction import org.apache.flink.table.runtime.stream.table.GroupWindowITCase._ import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithMerge, WeightedAvg, WeightedAvgWithMerge} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowTableAggregateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowTableAggregateITCase.scala index e71e266fa2001..948d27a063d6c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowTableAggregateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowTableAggregateITCase.scala @@ -21,10 +21,8 @@ package org.apache.flink.table.runtime.stream.table import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala.{StreamTableEnvironment, _} - -import java.math.BigDecimal import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.stream.table.GroupWindowITCase._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData} import org.apache.flink.table.utils.Top3 @@ -34,6 +32,8 @@ import org.apache.flink.types.Row import org.junit.Assert._ import org.junit.{Before, Test} +import java.math.BigDecimal + /** * We only test some aggregations until better testing of constructed DataStream * programs is possible. diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala index b5d2abac56bd4..9f0e4e47c3fae 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/JoinITCase.scala @@ -21,14 +21,15 @@ package org.apache.flink.table.runtime.stream.table import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, Tumble, Types} import org.apache.flink.table.expressions.utils.Func20 import org.apache.flink.table.functions.aggfunctions.CountAggFunction import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, WeightedAvg} import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase} import org.apache.flink.types.Row + import org.junit.Assert._ import org.junit.{Before, Test} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala index 179933bcb4ffc..04a766b0f88b1 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala @@ -24,12 +24,12 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.watermark.Watermark -import org.apache.flink.table.api.{EnvironmentSettings, Over} -import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithRetractAndReset, WeightedAvg} -import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.JavaFunc0 -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.functions.aggfunctions.CountAggFunction import org.apache.flink.table.runtime.stream.table.OverWindowITCase._ +import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithRetractAndReset, WeightedAvg} +import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.JavaFunc0 import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala index d383136ff93f2..5554b29ab5186 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.runtime.stream.table import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase} import org.apache.flink.table.utils.TableFunc0 import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala index 3450ff24892af..5c3affd814cd7 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala @@ -20,8 +20,8 @@ package org.apache.flink.table.runtime.stream.table import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.CommonTestData.NonPojo import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData} import org.apache.flink.test.util.AbstractTestBase diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableAggregateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableAggregateITCase.scala index 95635e769d669..54172391fb23c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableAggregateITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableAggregateITCase.scala @@ -18,10 +18,10 @@ package org.apache.flink.table.runtime.stream.table -import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ import org.apache.flink.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, ValidationException} +import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase} import org.apache.flink.table.utils.{Top3, Top3WithEmitRetractValue, Top3WithMapView} import org.apache.flink.types.Row diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala index 2ecc4ad574664..841131dab5e52 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala @@ -28,21 +28,22 @@ import org.apache.flink.streaming.api.datastream.{DataStream, DataStreamSink} import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, TableException, Tumble, Types} +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData} import org.apache.flink.table.sinks._ import org.apache.flink.table.utils.MemoryTableSourceSinkUtil import org.apache.flink.test.util.{AbstractTestBase, TestBaseUtils} import org.apache.flink.types.Row import org.apache.flink.util.Collector + import org.junit.Assert._ import org.junit.{Before, Test} + import java.io.File import java.lang.{Boolean => JBool} -import org.apache.flink.table.api.internal.TableEnvironmentInternal - import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala index 70608049e0a4e..421b11f5913b4 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala @@ -18,9 +18,6 @@ package org.apache.flink.table.runtime.stream.table -import java.lang.{Boolean => JBool, Integer => JInt, Long => JLong} - -import org.apache.calcite.runtime.SqlFunctions.{internalToTimestamp => toTimestamp} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.{GenericTypeInfo, RowTypeInfo} import org.apache.flink.api.scala._ @@ -29,20 +26,23 @@ import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JExecEnv} import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.{EnvironmentSettings, TableException, TableSchema, Tumble, Types} import org.apache.flink.table.runtime.utils.{CommonTestData, StreamITCase} import org.apache.flink.table.sources.StreamTableSource import org.apache.flink.table.utils._ import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.types.Row import org.apache.flink.util.Collector + +import org.apache.calcite.runtime.SqlFunctions.{internalToTimestamp => toTimestamp} import org.junit.Assert._ import org.junit.{Before, Test} +import java.lang.{Boolean => JBool, Integer => JInt, Long => JLong} + import scala.collection.JavaConverters._ -import scala.collection.mutable class TableSourceITCase extends AbstractTestBase { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala index 66b9c7f742d43..44e0bb952c276 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TableTestBase.scala @@ -25,10 +25,10 @@ import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.environment.LocalStreamEnvironment import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.apache.flink.table.api.bridge.java.internal.{BatchTableEnvironmentImpl => JavaBatchTableEnvironmentImpl, StreamTableEnvironmentImpl => JavaStreamTableEnvironmentImpl} +import org.apache.flink.table.api.bridge.scala._ +import org.apache.flink.table.api.bridge.scala.internal.{BatchTableEnvironmentImpl => ScalaBatchTableEnvironmentImpl, StreamTableEnvironmentImpl => ScalaStreamTableEnvironmentImpl} import org.apache.flink.table.api.internal.{TableEnvImpl, TableEnvironmentImpl, TableImpl, BatchTableEnvImpl => _} -import org.apache.flink.table.api.java.internal.{BatchTableEnvironmentImpl => JavaBatchTableEnvironmentImpl, StreamTableEnvironmentImpl => JavaStreamTableEnvironmentImpl} -import org.apache.flink.table.api.scala._ -import org.apache.flink.table.api.scala.internal.{BatchTableEnvironmentImpl => ScalaBatchTableEnvironmentImpl, StreamTableEnvironmentImpl => ScalaStreamTableEnvironmentImpl} import org.apache.flink.table.api.{ApiExpression, Table, TableConfig, TableSchema} import org.apache.flink.table.catalog.{CatalogManager, FunctionCatalog} import org.apache.flink.table.executor.StreamExecutor @@ -45,8 +45,8 @@ import org.junit.rules.ExpectedException import org.junit.{ComparisonFailure, Rule} import org.mockito.Mockito.{mock, when} -import _root_.scala.util.control.Breaks._ import scala.io.Source +import scala.util.control.Breaks._ /** * Test base for testing Table API / SQL plans. diff --git a/flink-walkthroughs/flink-walkthrough-table-java/src/main/resources/archetype-resources/src/main/java/SpendReport.java b/flink-walkthroughs/flink-walkthrough-table-java/src/main/resources/archetype-resources/src/main/java/SpendReport.java index 5b892ed44028d..936147aaa9a1a 100644 --- a/flink-walkthroughs/flink-walkthrough-table-java/src/main/resources/archetype-resources/src/main/java/SpendReport.java +++ b/flink-walkthroughs/flink-walkthrough-table-java/src/main/resources/archetype-resources/src/main/java/SpendReport.java @@ -20,7 +20,7 @@ import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.table.api.internal.TableEnvironmentInternal; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; import org.apache.flink.walkthrough.common.table.SpendReportTableSink; import org.apache.flink.walkthrough.common.table.BoundedTransactionTableSource; import org.apache.flink.walkthrough.common.table.TruncateDateToHour; diff --git a/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala b/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala index febfe413f9b32..3a93a3b855900 100644 --- a/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala +++ b/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala @@ -20,7 +20,6 @@ package ${package} import org.apache.flink.api.scala._ import org.apache.flink.table.api.internal.TableEnvironmentInternal -import org.apache.flink.table.api.scala._ import org.apache.flink.walkthrough.common.table._ object SpendReport { From 87a0358deb51cf55f455d0dd4cfd6bf8690b2e2e Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Mon, 18 May 2020 19:49:59 +0200 Subject: [PATCH 077/773] [FLINK-15947] Update docs with updated package structure. This closes #12232 --- docs/dev/table/common.md | 18 ++++++++++-------- docs/dev/table/common.zh.md | 18 ++++++++++-------- docs/dev/table/tableApi.md | 7 +++---- docs/dev/table/tableApi.zh.md | 7 +++---- docs/getting-started/walkthroughs/table_api.md | 6 +++--- .../walkthroughs/table_api.zh.md | 5 +++-- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docs/dev/table/common.md b/docs/dev/table/common.md index 907cb15ac1013..4dae5b7c9c3d1 100644 --- a/docs/dev/table/common.md +++ b/docs/dev/table/common.md @@ -157,7 +157,7 @@ If both planner jars are on the classpath (the default behavior), you should exp // ********************** import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; EnvironmentSettings fsSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build(); StreamExecutionEnvironment fsEnv = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -168,7 +168,7 @@ StreamTableEnvironment fsTableEnv = StreamTableEnvironment.create(fsEnv, fsSetti // FLINK BATCH QUERY // ****************** import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; ExecutionEnvironment fbEnv = ExecutionEnvironment.getExecutionEnvironment(); BatchTableEnvironment fbTableEnv = BatchTableEnvironment.create(fbEnv); @@ -178,7 +178,7 @@ BatchTableEnvironment fbTableEnv = BatchTableEnvironment.create(fbEnv); // ********************** import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment(); EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build(); @@ -205,7 +205,7 @@ TableEnvironment bbTableEnv = TableEnvironment.create(bbSettings); // ********************** import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment val fsSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build() val fsEnv = StreamExecutionEnvironment.getExecutionEnvironment @@ -216,7 +216,7 @@ val fsTableEnv = StreamTableEnvironment.create(fsEnv, fsSettings) // FLINK BATCH QUERY // ****************** import org.apache.flink.api.scala.ExecutionEnvironment -import org.apache.flink.table.api.scala.BatchTableEnvironment +import org.apache.flink.table.api.bridge.scala.BatchTableEnvironment val fbEnv = ExecutionEnvironment.getExecutionEnvironment val fbTableEnv = BatchTableEnvironment.create(fbEnv) @@ -226,7 +226,7 @@ val fbTableEnv = BatchTableEnvironment.create(fbEnv) // ********************** import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment val bsEnv = StreamExecutionEnvironment.getExecutionEnvironment val bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build() @@ -552,7 +552,9 @@ val revenue = orders // execute query {% endhighlight %} -**Note:** The Scala Table API uses Scala Symbols, which start with a single tick (`'`) to reference the attributes of a `Table`. The Table API uses Scala implicits. Make sure to import `org.apache.flink.api.scala._` and `org.apache.flink.table.api.scala._` in order to use Scala implicit conversions. +**Note:** The Scala Table API uses Scala String interpolation that starts with a dollar sign (`$`) to reference the attributes of a `Table`. The Table API uses Scala implicits. Make sure to import +* `org.apache.flink.table.api._` - for implicit expression conversions +* `org.apache.flink.api.scala._` and `org.apache.flink.table.api.bridge.scala._` if you want to convert from/to DataStream.

@@ -879,7 +881,7 @@ This interaction can be achieved by converting a `DataStream` or `DataSet` into ### Implicit Conversion for Scala -The Scala Table API features implicit conversions for the `DataSet`, `DataStream`, and `Table` classes. These conversions are enabled by importing the package `org.apache.flink.table.api.scala._` in addition to `org.apache.flink.api.scala._` for the Scala DataStream API. +The Scala Table API features implicit conversions for the `DataSet`, `DataStream`, and `Table` classes. These conversions are enabled by importing the package `org.apache.flink.table.api.bridge.scala._` in addition to `org.apache.flink.api.scala._` for the Scala DataStream API. ### Create a View from a DataStream or DataSet diff --git a/docs/dev/table/common.zh.md b/docs/dev/table/common.zh.md index 91aab8969485d..831c19f604407 100644 --- a/docs/dev/table/common.zh.md +++ b/docs/dev/table/common.zh.md @@ -157,7 +157,7 @@ table_env.execute("python_job") // ********************** import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; EnvironmentSettings fsSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build(); StreamExecutionEnvironment fsEnv = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -168,7 +168,7 @@ StreamTableEnvironment fsTableEnv = StreamTableEnvironment.create(fsEnv, fsSetti // FLINK BATCH QUERY // ****************** import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; ExecutionEnvironment fbEnv = ExecutionEnvironment.getExecutionEnvironment(); BatchTableEnvironment fbTableEnv = BatchTableEnvironment.create(fbEnv); @@ -178,7 +178,7 @@ BatchTableEnvironment fbTableEnv = BatchTableEnvironment.create(fbEnv); // ********************** import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment(); EnvironmentSettings bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build(); @@ -205,7 +205,7 @@ TableEnvironment bbTableEnv = TableEnvironment.create(bbSettings); // ********************** import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment val fsSettings = EnvironmentSettings.newInstance().useOldPlanner().inStreamingMode().build() val fsEnv = StreamExecutionEnvironment.getExecutionEnvironment @@ -216,7 +216,7 @@ val fsTableEnv = StreamTableEnvironment.create(fsEnv, fsSettings) // FLINK BATCH QUERY // ****************** import org.apache.flink.api.scala.ExecutionEnvironment -import org.apache.flink.table.api.scala.BatchTableEnvironment +import org.apache.flink.table.api.bridge.scala.BatchTableEnvironment val fbEnv = ExecutionEnvironment.getExecutionEnvironment val fbTableEnv = BatchTableEnvironment.create(fbEnv) @@ -226,7 +226,7 @@ val fbTableEnv = BatchTableEnvironment.create(fbEnv) // ********************** import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.table.api.EnvironmentSettings -import org.apache.flink.table.api.scala.StreamTableEnvironment +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment val bsEnv = StreamExecutionEnvironment.getExecutionEnvironment val bsSettings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build() @@ -531,7 +531,9 @@ val revenue = orders // execute query {% endhighlight %} -**Note:** The Scala Table API uses Scala Symbols, which start with a single tick (`'`) to reference the attributes of a `Table`. The Table API uses Scala implicits. Make sure to import `org.apache.flink.api.scala._` and `org.apache.flink.table.api.scala._` in order to use Scala implicit conversions. +**Note:** The Scala Table API uses Scala String interpolation that starts with a dollar sign (`$`) to reference the attributes of a `Table`. The Table API uses Scala implicits. Make sure to import +* `org.apache.flink.table.api._` - for implicit expression conversions +* `org.apache.flink.api.scala._` and `org.apache.flink.table.api.bridge.scala._` if you want to convert from/to DataStream.
@@ -858,7 +860,7 @@ Table API 和 SQL 可以被很容易地集成并嵌入到 [DataStream]({{ site.b ### Scala 隐式转换 -Scala Table API 含有对 `DataSet`、`DataStream` 和 `Table` 类的隐式转换。 通过为 Scala DataStream API 导入 `org.apache.flink.table.api.scala._` 包以及 `org.apache.flink.api.scala._` 包，可以启用这些转换。 +Scala Table API 含有对 `DataSet`、`DataStream` 和 `Table` 类的隐式转换。 通过为 Scala DataStream API 导入 `org.apache.flink.table.api.bridge.scala._` 包以及 `org.apache.flink.api.scala._` 包，可以启用这些转换。 ### 通过 DataSet 或 DataStream 创建`视图` diff --git a/docs/dev/table/tableApi.md b/docs/dev/table/tableApi.md index 71977bddc1415..e39af041a5f48 100644 --- a/docs/dev/table/tableApi.md +++ b/docs/dev/table/tableApi.md @@ -46,7 +46,6 @@ For the Expression DSL it is also necessary to import static `org.apache.flink.t {% highlight java %} import org.apache.flink.table.api.* -import org.apache.flink.table.api.java.* import static org.apache.flink.table.api.Expressions.* @@ -73,14 +72,14 @@ result.print();
-The Scala Table API is enabled by importing `org.apache.flink.api.scala._` and `org.apache.flink.table.api.scala._`. +The Scala Table API is enabled by importing `org.apache.flink.table.api._`, `org.apache.flink.api.scala._`, and `org.apache.flink.table.api.bridge.scala._` (for bridging to/from DataStream). The following example shows how a Scala Table API program is constructed. Table fields are referenced using Scala's String interpolation using a dollar character (`$`). {% highlight scala %} import org.apache.flink.api.scala._ import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ // environment configuration val env = ExecutionEnvironment.getExecutionEnvironment @@ -3158,6 +3157,6 @@ timeIndicator = fieldReference , "." , ( "proctime" | "rowtime" ) ; **Temporal intervals:** Temporal intervals can be represented as number of months (`Types.INTERVAL_MONTHS`) or number of milliseconds (`Types.INTERVAL_MILLIS`). Intervals of same type can be added or subtracted (e.g. `1.hour + 10.minutes`). Intervals of milliseconds can be added to time points (e.g. `"2016-08-10".toDate + 5.days`). -**Scala expressions:** Scala expressions use implicit conversions. Therefore, make sure to add the wildcard import `org.apache.flink.table.api.scala._` to your programs. In case a literal is not treated as an expression, use `.toExpr` such as `3.toExpr` to force a literal to be converted. +**Scala expressions:** Scala expressions use implicit conversions. Therefore, make sure to add the wildcard import `org.apache.flink.table.api._` to your programs. In case a literal is not treated as an expression, use `.toExpr` such as `3.toExpr` to force a literal to be converted. {% top %} diff --git a/docs/dev/table/tableApi.zh.md b/docs/dev/table/tableApi.zh.md index 25e2904ae414e..9db6055aa1a54 100644 --- a/docs/dev/table/tableApi.zh.md +++ b/docs/dev/table/tableApi.zh.md @@ -46,7 +46,6 @@ For the Expression DSL it is also necessary to import static `org.apache.flink.t {% highlight java %} import org.apache.flink.table.api.* -import org.apache.flink.table.api.java.* import static org.apache.flink.table.api.Expressions.* @@ -73,14 +72,14 @@ result.print();
-The Scala Table API is enabled by importing `org.apache.flink.api.scala._` and `org.apache.flink.table.api.scala._`. +The Scala Table API is enabled by importing `org.apache.flink.table.api._`, `org.apache.flink.api.scala._`, and `org.apache.flink.table.api.bridge.scala._` (for bridging to/from DataStream). The following example shows how a Scala Table API program is constructed. Table fields are referenced using Scala's String interpolation using a dollar character (`$`). {% highlight scala %} import org.apache.flink.api.scala._ import org.apache.flink.table.api._ -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.bridge.scala._ // environment configuration val env = ExecutionEnvironment.getExecutionEnvironment @@ -3157,6 +3156,6 @@ timeIndicator = fieldReference , "." , ( "proctime" | "rowtime" ) ; **Temporal intervals:** Temporal intervals can be represented as number of months (`Types.INTERVAL_MONTHS`) or number of milliseconds (`Types.INTERVAL_MILLIS`). Intervals of same type can be added or subtracted (e.g. `1.hour + 10.minutes`). Intervals of milliseconds can be added to time points (e.g. `"2016-08-10".toDate + 5.days`). -**Scala expressions:** Scala expressions use implicit conversions. Therefore, make sure to add the wildcard import `org.apache.flink.table.api.scala._` to your programs. In case a literal is not treated as an expression, use `.toExpr` such as `3.toExpr` to force a literal to be converted. +**Scala expressions:** Scala expressions use implicit conversions. Therefore, make sure to add the wildcard import `org.apache.flink.table.api._` to your programs. In case a literal is not treated as an expression, use `.toExpr` such as `3.toExpr` to force a literal to be converted. {% top %} diff --git a/docs/getting-started/walkthroughs/table_api.md b/docs/getting-started/walkthroughs/table_api.md index 0c99dbca10a1b..f1293fbb09f02 100644 --- a/docs/getting-started/walkthroughs/table_api.md +++ b/docs/getting-started/walkthroughs/table_api.md @@ -450,7 +450,7 @@ import org.apache.flink.walkthrough.common.table.TransactionTableSource; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.Tumble; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; public class SpendReport { @@ -482,8 +482,8 @@ package spendreport import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.TimeCharacteristic -import org.apache.flink.table.api.Tumble -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment; import org.apache.flink.walkthrough.common.table._ object SpendReport { diff --git a/docs/getting-started/walkthroughs/table_api.zh.md b/docs/getting-started/walkthroughs/table_api.zh.md index f6b23a4f63580..3194ae84637c0 100644 --- a/docs/getting-started/walkthroughs/table_api.zh.md +++ b/docs/getting-started/walkthroughs/table_api.zh.md @@ -451,7 +451,7 @@ import org.apache.flink.walkthrough.common.table.TransactionTableSource; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.Tumble; -import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; public class SpendReport { @@ -484,7 +484,8 @@ package spendreport import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.table.api.Tumble -import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api._ +import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment import org.apache.flink.walkthrough.common.table._ object SpendReport { From 90ece8c119ca1f748db324a301ac82a44ecda185 Mon Sep 17 00:00:00 2001 From: Arvid Heise Date: Sun, 17 May 2020 22:59:03 +0200 Subject: [PATCH 078/773] [FLINK-17780][checkpointing] Add task name to log statements of ChannelStateWriter. Add task name to the executor thread and to all method of ChannelStateWriter (as they can be called from any other thread), such that log statements can be connected to the respective task. --- .../ChannelStateWriteRequestExecutorImpl.java | 21 +++++--- .../channel/ChannelStateWriterImpl.java | 51 ++++++++++++------- ...nnelStateWriteRequestExecutorImplTest.java | 14 ++--- .../channel/ChannelStateWriterImplTest.java | 15 ++++-- .../state/ChannelPersistenceITCase.java | 2 +- .../SubtaskCheckpointCoordinatorImpl.java | 2 +- 6 files changed, 67 insertions(+), 38 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java index e87a21cadb500..3ad8982c924a7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImpl.java @@ -51,15 +51,20 @@ class ChannelStateWriteRequestExecutorImpl implements ChannelStateWriteRequestEx private final Thread thread; private volatile Exception thrown = null; private volatile boolean wasClosed = false; + private final String taskName; - ChannelStateWriteRequestExecutorImpl(ChannelStateWriteRequestDispatcher dispatcher) { - this(dispatcher, new LinkedBlockingDeque<>(DEFAULT_HANDOVER_CAPACITY)); + ChannelStateWriteRequestExecutorImpl(String taskName, ChannelStateWriteRequestDispatcher dispatcher) { + this(taskName, dispatcher, new LinkedBlockingDeque<>(DEFAULT_HANDOVER_CAPACITY)); } - ChannelStateWriteRequestExecutorImpl(ChannelStateWriteRequestDispatcher dispatcher, BlockingDeque deque) { + ChannelStateWriteRequestExecutorImpl( + String taskName, + ChannelStateWriteRequestDispatcher dispatcher, + BlockingDeque deque) { + this.taskName = taskName; this.dispatcher = dispatcher; this.deque = deque; - this.thread = new Thread(this::run); + this.thread = new Thread(this::run, "Channel state writer " + taskName); this.thread.setDaemon(true); } @@ -80,7 +85,7 @@ void run() { thrown = ExceptionUtils.firstOrSuppressed(e, thrown); } } - LOG.debug("loop terminated"); + LOG.debug("{} loop terminated", taskName); } private void loop() throws Exception { @@ -89,7 +94,7 @@ private void loop() throws Exception { dispatcher.dispatch(deque.take()); } catch (InterruptedException e) { if (!wasClosed) { - LOG.debug("interrupted while waiting for a request (continue waiting)", e); + LOG.debug(taskName + " interrupted while waiting for a request (continue waiting)", e); } else { Thread.currentThread().interrupt(); } @@ -101,7 +106,7 @@ private void cleanupRequests() throws Exception { Throwable cause = thrown == null ? new CancellationException() : thrown; List drained = new ArrayList<>(); deque.drainTo(drained); - LOG.info("discarding {} drained requests", drained.size()); + LOG.info("{} discarding {} drained requests", taskName, drained.size()); closeAll(drained.stream().map(request -> () -> request.cancel(cause)).collect(Collectors.toList())); } @@ -150,7 +155,7 @@ public void close() throws IOException { if (!thread.isAlive()) { Thread.currentThread().interrupt(); } - LOG.debug("interrupted while waiting for the writer thread to die", e); + LOG.debug(taskName + " interrupted while waiting for the writer thread to die", e); } } if (thrown != null) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java index b6fa58841adc6..3f56b1592e48d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImpl.java @@ -58,6 +58,7 @@ public class ChannelStateWriterImpl implements ChannelStateWriter { private static final Logger LOG = LoggerFactory.getLogger(ChannelStateWriterImpl.class); private static final int DEFAULT_MAX_CHECKPOINTS = 5; // currently, only single in-flight checkpoint is supported + private final String taskName; private final ChannelStateWriteRequestExecutor executor; private final ConcurrentMap results; private final int maxCheckpoints; @@ -65,26 +66,31 @@ public class ChannelStateWriterImpl implements ChannelStateWriter { /** * Creates a {@link ChannelStateWriterImpl} with {@link #DEFAULT_MAX_CHECKPOINTS} as {@link #maxCheckpoints}. */ - public ChannelStateWriterImpl(CheckpointStorageWorkerView streamFactoryResolver) { - this(streamFactoryResolver, DEFAULT_MAX_CHECKPOINTS); + public ChannelStateWriterImpl(String taskName, CheckpointStorageWorkerView streamFactoryResolver) { + this(taskName, streamFactoryResolver, DEFAULT_MAX_CHECKPOINTS); } /** * Creates a {@link ChannelStateWriterImpl} with {@link ChannelStateSerializerImpl default} {@link ChannelStateSerializer}, * and a {@link ChannelStateWriteRequestExecutorImpl}. - * - * @param maxCheckpoints maximum number of checkpoints to be written currently or finished but not taken yet. + * @param taskName * @param streamFactoryResolver a factory to obtain output stream factory for a given checkpoint + * @param maxCheckpoints maximum number of checkpoints to be written currently or finished but not taken yet. */ - ChannelStateWriterImpl(CheckpointStorageWorkerView streamFactoryResolver, int maxCheckpoints) { + ChannelStateWriterImpl(String taskName, CheckpointStorageWorkerView streamFactoryResolver, int maxCheckpoints) { this( + taskName, new ConcurrentHashMap<>(maxCheckpoints), - new ChannelStateWriteRequestExecutorImpl(new ChannelStateWriteRequestDispatcherImpl(streamFactoryResolver, new ChannelStateSerializerImpl())), - maxCheckpoints - ); + new ChannelStateWriteRequestExecutorImpl(taskName, new ChannelStateWriteRequestDispatcherImpl(streamFactoryResolver, new ChannelStateSerializerImpl())), + maxCheckpoints); } - ChannelStateWriterImpl(ConcurrentMap results, ChannelStateWriteRequestExecutor executor, int maxCheckpoints) { + ChannelStateWriterImpl( + String taskName, + ConcurrentMap results, + ChannelStateWriteRequestExecutor executor, + int maxCheckpoints) { + this.taskName = taskName; this.results = results; this.maxCheckpoints = maxCheckpoints; this.executor = executor; @@ -93,7 +99,7 @@ public ChannelStateWriterImpl(CheckpointStorageWorkerView streamFactoryResolver) @Override public void start(long checkpointId, CheckpointOptions checkpointOptions) { results.keySet().forEach(oldCheckpointId -> abort(oldCheckpointId, new Exception("Starting new checkpoint " + checkpointId))); - LOG.debug("start checkpoint {} ({})", checkpointId, checkpointOptions); + LOG.debug("{} starting checkpoint {} ({})", taskName, checkpointId, checkpointOptions); ChannelStateWriteResult result = new ChannelStateWriteResult(); ChannelStateWriteResult put = results.computeIfAbsent(checkpointId, id -> { Preconditions.checkState(results.size() < maxCheckpoints, "results.size() > maxCheckpoints", results.size(), maxCheckpoints); @@ -105,32 +111,42 @@ public void start(long checkpointId, CheckpointOptions checkpointOptions) { @Override public void addInputData(long checkpointId, InputChannelInfo info, int startSeqNum, CloseableIterator iterator) { - LOG.debug("add input data, checkpoint id: {}, channel: {}, startSeqNum: {}", checkpointId, info, startSeqNum); + LOG.debug( + "{} adding input data, checkpoint id: {}, channel: {}, startSeqNum: {}", + taskName, + checkpointId, + info, + startSeqNum); enqueue(write(checkpointId, info, iterator), false); } @Override public void addOutputData(long checkpointId, ResultSubpartitionInfo info, int startSeqNum, Buffer... data) { - LOG.debug("add output data, checkpoint id: {}, channel: {}, startSeqNum: {}, num buffers: {}", - checkpointId, info, startSeqNum, data == null ? 0 : data.length); + LOG.debug( + "{} adding output data, checkpoint id: {}, channel: {}, startSeqNum: {}, num buffers: {}", + taskName, + checkpointId, + info, + startSeqNum, + data == null ? 0 : data.length); enqueue(write(checkpointId, info, checkBufferType(data)), false); } @Override public void finishInput(long checkpointId) { - LOG.debug("finish input data, checkpoint id: {}", checkpointId); + LOG.debug("{} finishing input data, checkpoint id: {}", taskName, checkpointId); enqueue(completeInput(checkpointId), false); } @Override public void finishOutput(long checkpointId) { - LOG.debug("finish output data, checkpoint id: {}", checkpointId); + LOG.debug("{} finishing output data, checkpoint id: {}", taskName, checkpointId); enqueue(completeOutput(checkpointId), false); } @Override public void abort(long checkpointId, Throwable cause) { - LOG.debug("abort, checkpoint id: {}", checkpointId); + LOG.debug("{} aborting, checkpoint id: {}", taskName, checkpointId); enqueue(ChannelStateWriteRequest.abort(checkpointId, cause), true); // abort already started enqueue(ChannelStateWriteRequest.abort(checkpointId, cause), false); // abort enqueued but not started results.remove(checkpointId); @@ -138,7 +154,7 @@ public void abort(long checkpointId, Throwable cause) { @Override public ChannelStateWriteResult getWriteResult(long checkpointId) { - LOG.debug("requested write result, checkpoint id: {}", checkpointId); + LOG.debug("{} requested write result, checkpoint id: {}", taskName, checkpointId); ChannelStateWriteResult result = results.get(checkpointId); Preconditions.checkArgument(result != null, "channel state write result not found for checkpoint id " + checkpointId); return result; @@ -146,6 +162,7 @@ public ChannelStateWriteResult getWriteResult(long checkpointId) { @Override public void stop(long checkpointId) { + LOG.debug("{} stopping checkpoint id: {}", taskName, checkpointId); results.remove(checkpointId); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java index a299b34dc4c9e..431d967688594 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriteRequestExecutorImplTest.java @@ -39,6 +39,8 @@ */ public class ChannelStateWriteRequestExecutorImplTest { + private static final String TASK_NAME = "test task"; + @Test(expected = IllegalStateException.class) public void testCloseAfterSubmit() throws Exception { testCloseAfterSubmit(ChannelStateWriteRequestExecutor::submit); @@ -61,7 +63,7 @@ public void testSubmitPriorityFailure() throws Exception { private void testCloseAfterSubmit(BiConsumerWithException requestFun) throws Exception { WorkerClosingDeque closingDeque = new WorkerClosingDeque(); - ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(NO_OP, closingDeque); + ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(TASK_NAME, NO_OP, closingDeque); closingDeque.setWorker(worker); TestWriteRequest request = new TestWriteRequest(); requestFun.accept(worker, request); @@ -73,7 +75,7 @@ private void testSubmitFailure(BiConsumerWithException deque = new LinkedBlockingDeque<>(); try { - submitAction.accept(new ChannelStateWriteRequestExecutorImpl(NO_OP, deque), request); + submitAction.accept(new ChannelStateWriteRequestExecutorImpl(TASK_NAME, NO_OP, deque), request); } catch (IllegalStateException e) { // expected: executor not started; return; @@ -91,7 +93,7 @@ public void testCleanup() throws IOException { LinkedBlockingDeque deque = new LinkedBlockingDeque<>(); deque.add(request); TestRequestDispatcher requestProcessor = new TestRequestDispatcher(); - ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(requestProcessor, deque); + ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(TASK_NAME, requestProcessor, deque); worker.close(); worker.run(); @@ -105,7 +107,7 @@ public void testCleanup() throws IOException { public void testIgnoresInterruptsWhileRunning() throws Exception { TestRequestDispatcher requestProcessor = new TestRequestDispatcher(); LinkedBlockingDeque deque = new LinkedBlockingDeque<>(); - try (ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(requestProcessor, deque)) { + try (ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(TASK_NAME, requestProcessor, deque)) { worker.start(); worker.getThread().interrupt(); worker.submit(new TestWriteRequest()); @@ -119,7 +121,7 @@ public void testIgnoresInterruptsWhileRunning() throws Exception { @Test public void testCanBeClosed() throws IOException { TestRequestDispatcher requestProcessor = new TestRequestDispatcher(); - try (ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(requestProcessor)) { + try (ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(TASK_NAME, requestProcessor)) { worker.start(); } } @@ -134,7 +136,7 @@ public void dispatch(ChannelStateWriteRequest request) { } }; LinkedBlockingDeque deque = new LinkedBlockingDeque<>(Arrays.asList(new TestWriteRequest())); - ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(throwingRequestProcessor, deque); + ChannelStateWriteRequestExecutorImpl worker = new ChannelStateWriteRequestExecutorImpl(TASK_NAME, throwingRequestProcessor, deque); worker.run(); try { worker.close(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java index 92a7e881f57a5..8c7d7f266f46b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/ChannelStateWriterImplTest.java @@ -46,6 +46,7 @@ */ public class ChannelStateWriterImplTest { private static final long CHECKPOINT_ID = 42L; + private static final String TASK_NAME = "test"; @Test(expected = IllegalArgumentException.class) public void testAddEventBuffer() throws Exception { @@ -122,7 +123,11 @@ public void testAbortIgnoresMissing() throws Exception { public void testBuffersRecycledOnError() throws Exception { unwrappingError(TestException.class, () -> { NetworkBuffer buffer = getBuffer(); - try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl(new ConcurrentHashMap<>(), failingWorker(), 5)) { + try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl( + TASK_NAME, + new ConcurrentHashMap<>(), + failingWorker(), + 5)) { writer.open(); callAddInputData(writer, buffer); } finally { @@ -179,7 +184,7 @@ public void testRethrowOnClose() throws Exception { @Test(expected = TestException.class) public void testRethrowOnNextCall() throws Exception { SyncChannelStateWriteRequestExecutor worker = new SyncChannelStateWriteRequestExecutor(); - ChannelStateWriterImpl writer = new ChannelStateWriterImpl(new ConcurrentHashMap<>(), worker, 5); + ChannelStateWriterImpl writer = new ChannelStateWriterImpl(TASK_NAME, new ConcurrentHashMap<>(), worker, 5); writer.open(); worker.setThrown(new TestException()); unwrappingError(TestException.class, () -> callStart(writer)); @@ -203,7 +208,7 @@ public void testStartAbortsOldCheckpoints() throws Exception { @Test(expected = IllegalStateException.class) public void testStartNotOpened() throws Exception { unwrappingError(IllegalStateException.class, () -> { - try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl(getStreamFactoryFactory())) { + try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl(TASK_NAME, getStreamFactoryFactory())) { callStart(writer); } }); @@ -269,7 +274,7 @@ private void runWithSyncWorker(Consumer writerConsumer) thro private void runWithSyncWorker(BiConsumerWithException testFn) throws Exception { try ( SyncChannelStateWriteRequestExecutor worker = new SyncChannelStateWriteRequestExecutor(); - ChannelStateWriterImpl writer = new ChannelStateWriterImpl(new ConcurrentHashMap<>(), worker, 5) + ChannelStateWriterImpl writer = new ChannelStateWriterImpl(TASK_NAME, new ConcurrentHashMap<>(), worker, 5) ) { writer.open(); testFn.accept(writer, worker); @@ -278,7 +283,7 @@ private void runWithSyncWorker(BiConsumerWithException icBuffers = wrapWithBuffers(icMap); Map rsBuffers = wrapWithBuffers(rsMap); - try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl(getStreamFactoryFactory(maxStateSize))) { + try (ChannelStateWriterImpl writer = new ChannelStateWriterImpl("test", getStreamFactoryFactory(maxStateSize))) { writer.open(); writer.start(checkpointId, new CheckpointOptions(CHECKPOINT, new CheckpointStorageLocationReference("poly".getBytes()))); for (Map.Entry e : icBuffers.entrySet()) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java index 7508a16805726..9fc79272161d6 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SubtaskCheckpointCoordinatorImpl.java @@ -152,7 +152,7 @@ class SubtaskCheckpointCoordinatorImpl implements SubtaskCheckpointCoordinator { } private ChannelStateWriter openChannelStateWriter() { - ChannelStateWriterImpl writer = new ChannelStateWriterImpl(this.checkpointStorage); + ChannelStateWriterImpl writer = new ChannelStateWriterImpl(taskName, checkpointStorage); writer.open(); return writer; } From f7356560145f2bb862d1608264de3cf476f4abba Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Wed, 20 May 2020 22:22:46 +0800 Subject: [PATCH 079/773] [FLINK-16922][table-common] Fix DecimalData.toUnscaledBytes() should be consistent with BigDecimla.unscaledValue.toByteArray() This closes #12265 --- .../apache/flink/table/data/DecimalData.java | 26 +-- .../flink/table/data/DecimalDataTest.java | 154 ++++++++++-------- 2 files changed, 90 insertions(+), 90 deletions(-) diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/DecimalData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/DecimalData.java index 38bba78d7d08c..5f513bfb3bc46 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/DecimalData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/DecimalData.java @@ -138,18 +138,7 @@ public long toUnscaledLong() { * @return the unscaled byte array of this {@link DecimalData}. */ public byte[] toUnscaledBytes() { - if (!isCompact()) { - return toBigDecimal().unscaledValue().toByteArray(); - } - - // big endian; consistent with BigInteger.toByteArray() - byte[] bytes = new byte[8]; - long l = longVal; - for (int i = 0; i < 8; i++) { - bytes[7 - i] = (byte) l; - l >>>= 8; - } - return bytes; + return toBigDecimal().unscaledValue().toByteArray(); } /** @@ -231,17 +220,8 @@ public static DecimalData fromUnscaledLong(long unscaledLong, int precision, int * and scale. */ public static DecimalData fromUnscaledBytes(byte[] unscaledBytes, int precision, int scale) { - if (precision > MAX_COMPACT_PRECISION) { - BigDecimal bd = new BigDecimal(new BigInteger(unscaledBytes), scale); - return new DecimalData(precision, scale, -1, bd); - } - assert unscaledBytes.length == 8; - long l = 0; - for (int i = 0; i < 8; i++) { - l <<= 8; - l |= (unscaledBytes[i] & (0xff)); - } - return new DecimalData(precision, scale, l, null); + BigDecimal bd = new BigDecimal(new BigInteger(unscaledBytes), scale); + return fromBigDecimal(bd, precision, scale); } /** diff --git a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DecimalDataTest.java b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DecimalDataTest.java index bfa51ca2a401b..e05410427f2a5 100644 --- a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DecimalDataTest.java +++ b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/data/DecimalDataTest.java @@ -18,7 +18,6 @@ package org.apache.flink.table.data; -import org.junit.Assert; import org.junit.Test; import java.math.BigDecimal; @@ -46,6 +45,10 @@ import static org.apache.flink.table.data.DecimalDataUtils.signum; import static org.apache.flink.table.data.DecimalDataUtils.sround; import static org.apache.flink.table.data.DecimalDataUtils.subtract; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * Test for {@link DecimalData}. @@ -55,55 +58,72 @@ public class DecimalDataTest { @SuppressWarnings("ConstantConditions") @Test public void testNormal() { + BigDecimal bigDecimal1 = new BigDecimal("13145678.90123"); + BigDecimal bigDecimal2 = new BigDecimal("1234567890.0987654321"); + // fromUnscaledBytes + assertEquals( + DecimalData.fromBigDecimal(bigDecimal1, 15, 5), + DecimalData.fromUnscaledBytes(bigDecimal1.unscaledValue().toByteArray(), 15, 5)); + assertEquals( + DecimalData.fromBigDecimal(bigDecimal2, 23, 10), + DecimalData.fromUnscaledBytes(bigDecimal2.unscaledValue().toByteArray(), 23, 10)); + // toUnscaledBytes + assertArrayEquals( + bigDecimal1.unscaledValue().toByteArray(), + DecimalData.fromUnscaledBytes(bigDecimal1.unscaledValue().toByteArray(), 15, 5).toUnscaledBytes()); + assertArrayEquals( + bigDecimal2.unscaledValue().toByteArray(), + DecimalData.fromUnscaledBytes(bigDecimal2.unscaledValue().toByteArray(), 23, 10).toUnscaledBytes()); + DecimalData decimal1 = DecimalData.fromUnscaledLong(10, 5, 0); DecimalData decimal2 = DecimalData.fromUnscaledLong(15, 5, 0); - Assert.assertEquals(decimal1.hashCode(), + assertEquals(decimal1.hashCode(), DecimalData.fromBigDecimal(new BigDecimal(10), 5, 0).hashCode()); - Assert.assertEquals(decimal1, decimal1.copy()); - Assert.assertEquals(decimal1, DecimalData.fromUnscaledLong(decimal1.toUnscaledLong(), 5, 0)); - Assert.assertEquals(decimal1, DecimalData.fromUnscaledBytes(decimal1.toUnscaledBytes(), 5, 0)); - Assert.assertTrue(decimal1.compareTo(decimal2) < 0); - Assert.assertEquals(1, signum(decimal1)); - Assert.assertEquals(10.5, doubleValue(castFrom(10.5, 5, 1)), 0.0); - Assert.assertEquals(DecimalData.fromUnscaledLong(-10, 5, 0), negate(decimal1)); - Assert.assertEquals(decimal1, abs(decimal1)); - Assert.assertEquals(decimal1, abs(negate(decimal1))); - Assert.assertEquals(25, add(decimal1, decimal2, 5, 0).toUnscaledLong()); - Assert.assertEquals(-5, subtract(decimal1, decimal2, 5, 0).toUnscaledLong()); - Assert.assertEquals(150, multiply(decimal1, decimal2, 5, 0).toUnscaledLong()); - Assert.assertEquals(0.67, doubleValue(divide(decimal1, decimal2, 5, 2)), 0.0); - Assert.assertEquals(decimal1, mod(decimal1, decimal2, 5, 0)); - Assert.assertEquals(5, divideToIntegralValue( + assertEquals(decimal1, decimal1.copy()); + assertEquals(decimal1, DecimalData.fromUnscaledLong(decimal1.toUnscaledLong(), 5, 0)); + assertEquals(decimal1, DecimalData.fromUnscaledBytes(decimal1.toUnscaledBytes(), 5, 0)); + assertTrue(decimal1.compareTo(decimal2) < 0); + assertEquals(1, signum(decimal1)); + assertEquals(10.5, doubleValue(castFrom(10.5, 5, 1)), 0.0); + assertEquals(DecimalData.fromUnscaledLong(-10, 5, 0), negate(decimal1)); + assertEquals(decimal1, abs(decimal1)); + assertEquals(decimal1, abs(negate(decimal1))); + assertEquals(25, add(decimal1, decimal2, 5, 0).toUnscaledLong()); + assertEquals(-5, subtract(decimal1, decimal2, 5, 0).toUnscaledLong()); + assertEquals(150, multiply(decimal1, decimal2, 5, 0).toUnscaledLong()); + assertEquals(0.67, doubleValue(divide(decimal1, decimal2, 5, 2)), 0.0); + assertEquals(decimal1, mod(decimal1, decimal2, 5, 0)); + assertEquals(5, divideToIntegralValue( decimal1, DecimalData.fromUnscaledLong(2, 5, 0), 5, 0).toUnscaledLong()); - Assert.assertEquals(10, castToIntegral(decimal1)); - Assert.assertTrue(castToBoolean(decimal1)); - Assert.assertEquals(0, compare(decimal1, 10)); - Assert.assertTrue(compare(decimal1, 5) > 0); - Assert.assertEquals(castFrom(1.0, 10, 5), sign(castFrom(5.556, 10, 5))); + assertEquals(10, castToIntegral(decimal1)); + assertTrue(castToBoolean(decimal1)); + assertEquals(0, compare(decimal1, 10)); + assertTrue(compare(decimal1, 5) > 0); + assertEquals(castFrom(1.0, 10, 5), sign(castFrom(5.556, 10, 5))); - Assert.assertNull(DecimalData.fromBigDecimal(new BigDecimal(Long.MAX_VALUE), 5, 0)); - Assert.assertEquals(0, DecimalData.zero(5, 2).toBigDecimal().intValue()); - Assert.assertEquals(0, DecimalData.zero(20, 2).toBigDecimal().intValue()); + assertNull(DecimalData.fromBigDecimal(new BigDecimal(Long.MAX_VALUE), 5, 0)); + assertEquals(0, DecimalData.zero(5, 2).toBigDecimal().intValue()); + assertEquals(0, DecimalData.zero(20, 2).toBigDecimal().intValue()); - Assert.assertEquals(DecimalData.fromUnscaledLong(10, 5, 0), floor(castFrom(10.5, 5, 1))); - Assert.assertEquals(DecimalData.fromUnscaledLong(11, 5, 0), ceil(castFrom(10.5, 5, 1))); - Assert.assertEquals("5.00", castToDecimal(castFrom(5.0, 10, 1), 10, 2).toString()); + assertEquals(DecimalData.fromUnscaledLong(10, 5, 0), floor(castFrom(10.5, 5, 1))); + assertEquals(DecimalData.fromUnscaledLong(11, 5, 0), ceil(castFrom(10.5, 5, 1))); + assertEquals("5.00", castToDecimal(castFrom(5.0, 10, 1), 10, 2).toString()); - Assert.assertTrue(castToBoolean(castFrom(true, 5, 0))); - Assert.assertEquals(5, castToIntegral(castFrom(5, 5, 0))); - Assert.assertEquals(5, castToIntegral(castFrom("5", 5, 0))); - Assert.assertEquals(5000, castToTimestamp(castFrom("5", 5, 0))); + assertTrue(castToBoolean(castFrom(true, 5, 0))); + assertEquals(5, castToIntegral(castFrom(5, 5, 0))); + assertEquals(5, castToIntegral(castFrom("5", 5, 0))); + assertEquals(5000, castToTimestamp(castFrom("5", 5, 0))); DecimalData newDecimal = castFrom(castFrom(10, 5, 2), 10, 4); - Assert.assertEquals(10, newDecimal.precision()); - Assert.assertEquals(4, newDecimal.scale()); + assertEquals(10, newDecimal.precision()); + assertEquals(4, newDecimal.scale()); - Assert.assertTrue(is32BitDecimal(6)); - Assert.assertTrue(is64BitDecimal(11)); - Assert.assertTrue(isByteArrayDecimal(20)); + assertTrue(is32BitDecimal(6)); + assertTrue(is64BitDecimal(11)); + assertTrue(isByteArrayDecimal(20)); - Assert.assertEquals(6, sround(castFrom(5.555, 5, 0), 1).toUnscaledLong()); - Assert.assertEquals(56, sround(castFrom(5.555, 5, 3), 1).toUnscaledLong()); + assertEquals(6, sround(castFrom(5.555, 5, 0), 1).toUnscaledLong()); + assertEquals(56, sround(castFrom(5.555, 5, 3), 1).toUnscaledLong()); } @SuppressWarnings("ConstantConditions") @@ -111,41 +131,41 @@ public void testNormal() { public void testNotCompact() { DecimalData decimal1 = DecimalData.fromBigDecimal(new BigDecimal(10), 20, 0); DecimalData decimal2 = DecimalData.fromBigDecimal(new BigDecimal(15), 20, 0); - Assert.assertEquals(decimal1.hashCode(), + assertEquals(decimal1.hashCode(), DecimalData.fromBigDecimal(new BigDecimal(10), 20, 0).hashCode()); - Assert.assertEquals(decimal1, decimal1.copy()); - Assert.assertEquals(decimal1, DecimalData.fromBigDecimal(decimal1.toBigDecimal(), 20, 0)); - Assert.assertEquals(decimal1, DecimalData.fromUnscaledBytes(decimal1.toUnscaledBytes(), 20, 0)); - Assert.assertTrue(decimal1.compareTo(decimal2) < 0); - Assert.assertEquals(1, signum(decimal1)); - Assert.assertEquals(10.5, doubleValue(castFrom(10.5, 20, 1)), 0.0); - Assert.assertEquals(DecimalData.fromBigDecimal(new BigDecimal(-10), 20, 0), negate(decimal1)); - Assert.assertEquals(decimal1, abs(decimal1)); - Assert.assertEquals(decimal1, abs(negate(decimal1))); - Assert.assertEquals(25, add(decimal1, decimal2, 20, 0).toBigDecimal().longValue()); - Assert.assertEquals(-5, subtract(decimal1, decimal2, 20, 0).toBigDecimal().longValue()); - Assert.assertEquals(150, multiply(decimal1, decimal2, 20, 0).toBigDecimal().longValue()); - Assert.assertEquals(0.67, doubleValue(divide(decimal1, decimal2, 20, 2)), 0.0); - Assert.assertEquals(decimal1, mod(decimal1, decimal2, 20, 0)); - Assert.assertEquals(5, divideToIntegralValue( + assertEquals(decimal1, decimal1.copy()); + assertEquals(decimal1, DecimalData.fromBigDecimal(decimal1.toBigDecimal(), 20, 0)); + assertEquals(decimal1, DecimalData.fromUnscaledBytes(decimal1.toUnscaledBytes(), 20, 0)); + assertTrue(decimal1.compareTo(decimal2) < 0); + assertEquals(1, signum(decimal1)); + assertEquals(10.5, doubleValue(castFrom(10.5, 20, 1)), 0.0); + assertEquals(DecimalData.fromBigDecimal(new BigDecimal(-10), 20, 0), negate(decimal1)); + assertEquals(decimal1, abs(decimal1)); + assertEquals(decimal1, abs(negate(decimal1))); + assertEquals(25, add(decimal1, decimal2, 20, 0).toBigDecimal().longValue()); + assertEquals(-5, subtract(decimal1, decimal2, 20, 0).toBigDecimal().longValue()); + assertEquals(150, multiply(decimal1, decimal2, 20, 0).toBigDecimal().longValue()); + assertEquals(0.67, doubleValue(divide(decimal1, decimal2, 20, 2)), 0.0); + assertEquals(decimal1, mod(decimal1, decimal2, 20, 0)); + assertEquals(5, divideToIntegralValue( decimal1, DecimalData.fromBigDecimal(new BigDecimal(2), 20, 0), 20, 0).toBigDecimal().longValue()); - Assert.assertEquals(10, castToIntegral(decimal1)); - Assert.assertTrue(castToBoolean(decimal1)); - Assert.assertEquals(0, compare(decimal1, 10)); - Assert.assertTrue(compare(decimal1, 5) > 0); - Assert.assertTrue(compare(DecimalData.fromBigDecimal(new BigDecimal("10.5"), 20, 2), 10) > 0); - Assert.assertEquals(castFrom(1.0, 20, 5), sign(castFrom(5.556, 20, 5))); - - Assert.assertNull(DecimalData.fromBigDecimal(new BigDecimal(Long.MAX_VALUE), 5, 0)); - Assert.assertEquals(0, DecimalData.zero(20, 2).toBigDecimal().intValue()); - Assert.assertEquals(0, DecimalData.zero(20, 2).toBigDecimal().intValue()); + assertEquals(10, castToIntegral(decimal1)); + assertTrue(castToBoolean(decimal1)); + assertEquals(0, compare(decimal1, 10)); + assertTrue(compare(decimal1, 5) > 0); + assertTrue(compare(DecimalData.fromBigDecimal(new BigDecimal("10.5"), 20, 2), 10) > 0); + assertEquals(castFrom(1.0, 20, 5), sign(castFrom(5.556, 20, 5))); + + assertNull(DecimalData.fromBigDecimal(new BigDecimal(Long.MAX_VALUE), 5, 0)); + assertEquals(0, DecimalData.zero(20, 2).toBigDecimal().intValue()); + assertEquals(0, DecimalData.zero(20, 2).toBigDecimal().intValue()); } @Test public void testToString() { String val = "0.0000000000000000001"; - Assert.assertEquals(val, castFrom(val, 39, val.length() - 2).toString()); + assertEquals(val, castFrom(val, 39, val.length() - 2).toString()); val = "123456789012345678901234567890123456789"; - Assert.assertEquals(val, castFrom(val, 39, 0).toString()); + assertEquals(val, castFrom(val, 39, 0).toString()); } } From a83ee6c90b605f0807a40c82f2f5879f80f1f2dd Mon Sep 17 00:00:00 2001 From: Flavio Pompermaier Date: Mon, 18 May 2020 00:24:45 +0200 Subject: [PATCH 080/773] [FLINK-17356][jdbc][postgres] Support PK and Unique constraints This closes #11906 --- .../jdbc/catalog/AbstractJdbcCatalog.java | 36 +++++++++++++++++++ .../jdbc/catalog/PostgresCatalog.java | 14 +++++++- .../jdbc/catalog/PostgresTablePath.java | 8 +++++ .../jdbc/catalog/PostgresCatalogTestBase.java | 8 +++-- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java index 9e27816ee8951..199d4818a00fe 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java @@ -20,6 +20,7 @@ import org.apache.flink.connector.jdbc.table.JdbcDynamicTableSourceSinkFactory; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.constraints.UniqueConstraint; import org.apache.flink.table.catalog.AbstractCatalog; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogDatabase; @@ -50,11 +51,18 @@ import org.slf4j.LoggerFactory; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; +import java.util.AbstractMap; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkArgument; @@ -116,6 +124,34 @@ public String getBaseUrl() { return baseUrl; } + // ------ retrieve PK constraint ------ + + protected UniqueConstraint getPrimaryKey(DatabaseMetaData metaData, String schema, String table) throws SQLException { + + // According to the Javadoc of java.sql.DatabaseMetaData#getPrimaryKeys, + // the returned primary key columns are ordered by COLUMN_NAME, not by KEY_SEQ. + // We need to sort them based on the KEY_SEQ value. + ResultSet rs = metaData.getPrimaryKeys(null, schema, table); + + List> columnsWithIndex = null; + String pkName = null; + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + pkName = rs.getString("PK_NAME"); + int keySeq = rs.getInt("KEY_SEQ"); + if (columnsWithIndex == null) { + columnsWithIndex = new ArrayList<>(); + } + columnsWithIndex.add(new AbstractMap.SimpleEntry<>(Integer.valueOf(keySeq), columnName)); + } + if (columnsWithIndex != null) { + // sort columns by KEY_SEQ + columnsWithIndex.sort(Comparator.comparingInt(Map.Entry::getKey)); + List cols = columnsWithIndex.stream().map(Map.Entry::getValue).collect(Collectors.toList()); + return UniqueConstraint.primaryKey(pkName, cols); + } + return null; + } // ------ table factory ------ diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java index 31b4185f0069c..cb20fb07f28f3 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.constraints.UniqueConstraint; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogDatabase; import org.apache.flink.table.catalog.CatalogDatabaseImpl; @@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -180,6 +182,8 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep String dbUrl = baseUrl + tablePath.getDatabaseName(); try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd)) { + DatabaseMetaData metaData = conn.getMetaData(); + UniqueConstraint pk = getPrimaryKey(metaData, pgPath.getPgSchemaName(), pgPath.getPgTableName()); PreparedStatement ps = conn.prepareStatement( String.format("SELECT * FROM %s;", pgPath.getFullPath())); @@ -192,9 +196,17 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep for (int i = 1; i <= rsmd.getColumnCount(); i++) { names[i - 1] = rsmd.getColumnName(i); types[i - 1] = fromJDBCType(rsmd, i); + if (rsmd.isNullable(i) == ResultSetMetaData.columnNoNulls) { + types[i - 1] = types[i - 1].notNull(); + } } - TableSchema tableSchema = new TableSchema.Builder().fields(names, types).build(); + TableSchema.Builder tableBuilder = new TableSchema.Builder() + .fields(names, types); + if (pk != null) { + tableBuilder.primaryKey(pk.getName(), pk.getColumns().toArray(new String[0])); + } + TableSchema tableSchema = tableBuilder.build(); Map props = new HashMap<>(); props.put(CONNECTOR.key(), IDENTIFIER); diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresTablePath.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresTablePath.java index a9890243b6437..a2668673413cc 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresTablePath.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresTablePath.java @@ -64,6 +64,14 @@ public String getFullPath() { return String.format("%s.%s", pgSchemaName, pgTableName); } + public String getPgTableName() { + return pgTableName; + } + + public String getPgSchemaName() { + return pgSchemaName; + } + @Override public String toString() { return getFullPath(); diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java index b4b1b444851b1..fd916e74a3f28 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java @@ -157,9 +157,9 @@ public static TestTable getSimpleTable() { public static TestTable getPrimitiveTable() { return new TestTable( TableSchema.builder() - .field("int", DataTypes.INT()) + .field("int", DataTypes.INT().notNull()) .field("bytea", DataTypes.BYTES()) - .field("short", DataTypes.SMALLINT()) + .field("short", DataTypes.SMALLINT().notNull()) .field("long", DataTypes.BIGINT()) .field("real", DataTypes.FLOAT()) .field("double_precision", DataTypes.DOUBLE()) @@ -175,6 +175,7 @@ public static TestTable getPrimitiveTable() { .field("date", DataTypes.DATE()) .field("time", DataTypes.TIME(0)) .field("default_numeric", DataTypes.DECIMAL(DecimalType.MAX_PRECISION, 18)) + .primaryKey("test_pk", new String[]{"int", "short"}) .build(), "int integer, " + "bytea bytea, " + @@ -193,7 +194,8 @@ public static TestTable getPrimitiveTable() { // "timestamptz timestamptz(4), " + "date date," + "time time(0), " + - "default_numeric numeric ", + "default_numeric numeric, " + + "CONSTRAINT test_pk PRIMARY KEY (int, short)", "1," + "'2'," + "3," + From 99591cfe35d66d16798b7b2c5c42f11bba96b3af Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Wed, 20 May 2020 12:05:16 +0800 Subject: [PATCH 081/773] [hotfix][table-planner-blink] Fix code generation error for CAST STRING to BYTES --- .../flink/table/planner/codegen/calls/ScalarOperatorGens.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala index 6bcbfc6e10605..ff91ed2772473 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala @@ -1073,7 +1073,7 @@ object ScalarOperatorGens { // String -> binary case (VARCHAR | CHAR, VARBINARY | BINARY) => generateUnaryOperatorIfNotNull(ctx, targetType, operand) { - operandTerm => s"$operandTerm.getBytes()" + operandTerm => s"$operandTerm.toBytes()" } // Note: SQL2003 $6.12 - casting is not allowed between boolean and numeric types. From fa3768a82fd880178f5c8cb71c28510dd4db4d30 Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Wed, 20 May 2020 12:07:00 +0800 Subject: [PATCH 082/773] [FLINK-17356][jdbc][postgres] Add IT cases for inserting group by query into posgres catalog table --- .../jdbc/catalog/AbstractJdbcCatalog.java | 32 ++++++++---------- .../jdbc/catalog/PostgresCatalog.java | 12 ++++--- .../jdbc/catalog/PostgresCatalogITCase.java | 33 +++++++++++++++---- .../jdbc/catalog/PostgresCatalogTest.java | 6 +++- .../jdbc/catalog/PostgresCatalogTestBase.java | 18 +++++++--- 5 files changed, 67 insertions(+), 34 deletions(-) diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java index 199d4818a00fe..33603d2e4c7c1 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/AbstractJdbcCatalog.java @@ -55,14 +55,12 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.AbstractMap; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkArgument; @@ -126,31 +124,29 @@ public String getBaseUrl() { // ------ retrieve PK constraint ------ - protected UniqueConstraint getPrimaryKey(DatabaseMetaData metaData, String schema, String table) throws SQLException { + protected Optional getPrimaryKey(DatabaseMetaData metaData, String schema, String table) throws SQLException { // According to the Javadoc of java.sql.DatabaseMetaData#getPrimaryKeys, // the returned primary key columns are ordered by COLUMN_NAME, not by KEY_SEQ. // We need to sort them based on the KEY_SEQ value. ResultSet rs = metaData.getPrimaryKeys(null, schema, table); - List> columnsWithIndex = null; + Map keySeqColumnName = new HashMap<>(); String pkName = null; - while (rs.next()) { + while (rs.next()) { String columnName = rs.getString("COLUMN_NAME"); - pkName = rs.getString("PK_NAME"); + pkName = rs.getString("PK_NAME"); // all the PK_NAME should be the same int keySeq = rs.getInt("KEY_SEQ"); - if (columnsWithIndex == null) { - columnsWithIndex = new ArrayList<>(); - } - columnsWithIndex.add(new AbstractMap.SimpleEntry<>(Integer.valueOf(keySeq), columnName)); + keySeqColumnName.put(keySeq - 1, columnName); // KEY_SEQ is 1-based index } - if (columnsWithIndex != null) { - // sort columns by KEY_SEQ - columnsWithIndex.sort(Comparator.comparingInt(Map.Entry::getKey)); - List cols = columnsWithIndex.stream().map(Map.Entry::getValue).collect(Collectors.toList()); - return UniqueConstraint.primaryKey(pkName, cols); + List pkFields = Arrays.asList(new String[keySeqColumnName.size()]); // initialize size + keySeqColumnName.forEach(pkFields::set); + if (!pkFields.isEmpty()) { + // PK_NAME maybe null according to the javadoc, generate an unique name in that case + pkName = pkName != null ? pkName : "pk_" + String.join("_", pkFields); + return Optional.of(UniqueConstraint.primaryKey(pkName, pkFields)); } - return null; + return Optional.empty(); } // ------ table factory ------ diff --git a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java index cb20fb07f28f3..a8bb4b05747ba 100644 --- a/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java +++ b/flink-connectors/flink-connector-jdbc/src/main/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalog.java @@ -49,6 +49,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static org.apache.flink.connector.jdbc.table.JdbcDynamicTableSourceSinkFactory.IDENTIFIER; @@ -183,7 +184,10 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep String dbUrl = baseUrl + tablePath.getDatabaseName(); try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd)) { DatabaseMetaData metaData = conn.getMetaData(); - UniqueConstraint pk = getPrimaryKey(metaData, pgPath.getPgSchemaName(), pgPath.getPgTableName()); + Optional primaryKey = getPrimaryKey( + metaData, + pgPath.getPgSchemaName(), + pgPath.getPgTableName()); PreparedStatement ps = conn.prepareStatement( String.format("SELECT * FROM %s;", pgPath.getFullPath())); @@ -203,9 +207,9 @@ public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistExcep TableSchema.Builder tableBuilder = new TableSchema.Builder() .fields(names, types); - if (pk != null) { - tableBuilder.primaryKey(pk.getName(), pk.getColumns().toArray(new String[0])); - } + primaryKey.ifPresent(pk -> + tableBuilder.primaryKey(pk.getName(), pk.getColumns().toArray(new String[0])) + ); TableSchema tableSchema = tableBuilder.build(); Map props = new HashMap<>(); diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java index a5ad1ec463bc7..5defb2e252e45 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogITCase.java @@ -20,7 +20,7 @@ import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.planner.runtime.utils.TableEnvUtil; import org.apache.flink.types.Row; import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; @@ -69,18 +69,39 @@ public void test_fullPath() throws Exception { } @Test - public void test_insert() throws Exception { + public void testInsert() { TableEnvironment tEnv = getTableEnvWithPgCatalog(); - TableResult tableResult = tEnv.executeSql(String.format("insert into %s select * from `%s`", TABLE4, TABLE1)); - // wait to finish - tableResult.getJobClient().get().getJobExecutionResult(Thread.currentThread().getContextClassLoader()).get(); + TableEnvUtil.execInsertSqlAndWaitResult( + tEnv, + String.format("insert into %s select * from `%s`", TABLE4, TABLE1)); List results = Lists.newArrayList( - tEnv.sqlQuery(String.format("select * from %s", TABLE1)).execute().collect()); + tEnv.sqlQuery(String.format("select * from %s", TABLE4)).execute().collect()); assertEquals("[1]", results.toString()); } + @Test + public void testGroupByInsert() { + TableEnvironment tEnv = getTableEnvWithPgCatalog(); + + TableEnvUtil.execInsertSqlAndWaitResult( + tEnv, + String.format( + "insert into `%s` " + + "select `int`, cast('A' as bytes), `short`, max(`long`), max(`real`), " + + "max(`double_precision`), max(`numeric`), max(`boolean`), max(`text`), " + + "'B', 'C', max(`character_varying`), " + + "max(`timestamp`), max(`date`), max(`time`), max(`default_numeric`) " + + "from `%s` group by `int`, `short`", + TABLE_PRIMITIVE_TYPE2, + TABLE_PRIMITIVE_TYPE)); + + List results = Lists.newArrayList( + tEnv.sqlQuery(String.format("select * from `%s`", TABLE_PRIMITIVE_TYPE2)).execute().collect()); + assertEquals("[1,[65],3,4,5.5,6.6,7.70000,true,a,B,C ,d,2016-06-22T19:10:25,2015-01-01,00:51:03,500.000000000000000000]", results.toString()); + } + @Test public void testPrimitiveTypes() throws Exception { TableEnvironment tEnv = getTableEnvWithPgCatalog(); diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTest.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTest.java index 7787cc54b4377..7142d417456fb 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTest.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTest.java @@ -70,7 +70,11 @@ public void testDbExists() throws Exception { public void testListTables() throws DatabaseNotExistException { List actual = catalog.listTables(PostgresCatalog.DEFAULT_DATABASE); - assertEquals(Arrays.asList("public.dt", "public.dt2", "public.t1", "public.t4", "public.t5"), actual); + assertEquals( + Arrays.asList( + "public.array_table", "public.primitive_table", "public.primitive_table2", + "public.t1", "public.t4", "public.t5"), + actual); actual = catalog.listTables(TEST_DB); diff --git a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java index fd916e74a3f28..96606cc8741af 100644 --- a/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java +++ b/flink-connectors/flink-connector-jdbc/src/test/java/org/apache/flink/connector/jdbc/catalog/PostgresCatalogTestBase.java @@ -55,8 +55,9 @@ public class PostgresCatalogTestBase { protected static final String TABLE3 = "t3"; protected static final String TABLE4 = "t4"; protected static final String TABLE5 = "t5"; - protected static final String TABLE_PRIMITIVE_TYPE = "dt"; - protected static final String TABLE_ARRAY_TYPE = "dt2"; + protected static final String TABLE_PRIMITIVE_TYPE = "primitive_table"; + protected static final String TABLE_PRIMITIVE_TYPE2 = "primitive_table2"; + protected static final String TABLE_ARRAY_TYPE = "array_table"; protected static String baseUrl; protected static PostgresCatalog catalog; @@ -89,6 +90,7 @@ public static void init() throws SQLException { createTable(TEST_DB, PostgresTablePath.fromFlinkTableName(TABLE2), getSimpleTable().pgSchemaSql); createTable(TEST_DB, new PostgresTablePath(TEST_SCHEMA, TABLE3), getSimpleTable().pgSchemaSql); createTable(PostgresTablePath.fromFlinkTableName(TABLE_PRIMITIVE_TYPE), getPrimitiveTable().pgSchemaSql); + createTable(PostgresTablePath.fromFlinkTableName(TABLE_PRIMITIVE_TYPE2), getPrimitiveTable("test_pk2").pgSchemaSql); createTable(PostgresTablePath.fromFlinkTableName(TABLE_ARRAY_TYPE), getArrayTable().pgSchemaSql); executeSQL(PostgresCatalog.DEFAULT_DATABASE, String.format("insert into public.%s values (%s);", TABLE1, getSimpleTable().values)); @@ -150,11 +152,17 @@ public static TestTable getSimpleTable() { ); } + // posgres doesn't support to use the same primary key name across different tables, + // make the table parameterized to resolve this problem. + public static TestTable getPrimitiveTable() { + return getPrimitiveTable("test_pk"); + } + // TODO: add back timestamptz and time types. // Flink currently doens't support converting time's precision, with the following error // TableException: Unsupported conversion from data type 'TIME(6)' (conversion class: java.sql.Time) // to type information. Only data types that originated from type information fully support a reverse conversion. - public static TestTable getPrimitiveTable() { + public static TestTable getPrimitiveTable(String primaryKeyName) { return new TestTable( TableSchema.builder() .field("int", DataTypes.INT().notNull()) @@ -175,7 +183,7 @@ public static TestTable getPrimitiveTable() { .field("date", DataTypes.DATE()) .field("time", DataTypes.TIME(0)) .field("default_numeric", DataTypes.DECIMAL(DecimalType.MAX_PRECISION, 18)) - .primaryKey("test_pk", new String[]{"int", "short"}) + .primaryKey(primaryKeyName, new String[]{"short", "int"}) .build(), "int integer, " + "bytea bytea, " + @@ -195,7 +203,7 @@ public static TestTable getPrimitiveTable() { "date date," + "time time(0), " + "default_numeric numeric, " + - "CONSTRAINT test_pk PRIMARY KEY (int, short)", + "CONSTRAINT " + primaryKeyName + " PRIMARY KEY (short, int)", "1," + "'2'," + "3," + From 8044166877efc42c42a80344992d66ba18748a4d Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Wed, 20 May 2020 17:04:52 +0200 Subject: [PATCH 083/773] [FLINK-17846][table, e2e] Fix import in flink-walkthrough-table-scala --- .../archetype-resources/src/main/scala/SpendReport.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala b/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala index 3a93a3b855900..95e6149fdd957 100644 --- a/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala +++ b/flink-walkthroughs/flink-walkthrough-table-scala/src/main/resources/archetype-resources/src/main/scala/SpendReport.scala @@ -19,6 +19,7 @@ package ${package} import org.apache.flink.api.scala._ +import org.apache.flink.table.api.bridge.scala.BatchTableEnvironment import org.apache.flink.table.api.internal.TableEnvironmentInternal import org.apache.flink.walkthrough.common.table._ From 861efd05e382fa122520ab253f4278fa37bb2bad Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Mon, 18 May 2020 15:20:52 +0200 Subject: [PATCH 084/773] [FLINK-17675][docs] Update jquery dependency to 3.5.1 This closes #12229 --- NOTICE | 6 ++++++ docs/page/js/flink.js | 2 +- docs/page/js/jquery.min.js | 6 ++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/NOTICE b/NOTICE index 707f8d92f3a83..2179dc8f61c58 100644 --- a/NOTICE +++ b/NOTICE @@ -4,6 +4,12 @@ Copyright 2014-2020 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +This project bundles the following dependencies under the MIT license. +See bundled license files for details. + +- jQuery v3.5.1 | (c) JS Foundation and other contributors + -> in "docs/page/js/jquery.min.js" + This project bundles the following dependencies under the BSD license. See bundled license files for details. diff --git a/docs/page/js/flink.js b/docs/page/js/flink.js index 885a8ffad49b9..5d744e70c600c 100644 --- a/docs/page/js/flink.js +++ b/docs/page/js/flink.js @@ -123,5 +123,5 @@ $(function() { // Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers // will try to do *their* initial scroll after running the onReady handler. - $(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); }); + $(window).ready(function() { setTimeout(function() { maybeScrollToHash(); }, 25); }); }); diff --git a/docs/page/js/jquery.min.js b/docs/page/js/jquery.min.js index e6a051d0d1d32..b0614034ad3a9 100644 --- a/docs/page/js/jquery.min.js +++ b/docs/page/js/jquery.min.js @@ -1,4 +1,2 @@ -/*! jQuery v1.11.2 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ -!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l="1.11.2",m=function(a,b){return new m.fn.init(a,b)},n=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,o=/^-ms-/,p=/-([\da-z])/gi,q=function(a,b){return b.toUpperCase()};m.fn=m.prototype={jquery:l,constructor:m,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=m.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return m.each(this,a,b)},map:function(a){return this.pushStack(m.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},m.extend=m.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||m.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(m.isPlainObject(c)||(b=m.isArray(c)))?(b?(b=!1,f=a&&m.isArray(a)?a:[]):f=a&&m.isPlainObject(a)?a:{},g[d]=m.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},m.extend({expando:"jQuery"+(l+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===m.type(a)},isArray:Array.isArray||function(a){return"array"===m.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return!m.isArray(a)&&a-parseFloat(a)+1>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==m.type(a)||a.nodeType||m.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(k.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&m.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(o,"ms-").replace(p,q)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=r(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(n,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(r(Object(a))?m.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=r(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),m.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||m.guid++,e):void 0},now:function(){return+new Date},support:k}),m.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function r(a){var b=a.length,c=m.type(a);return"function"===c||m.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var s=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=hb(),z=hb(),A=hb(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N=M.replace("w","w#"),O="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+N+"))|)"+L+"*\\]",P=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+O+")*)|.*)\\)|)",Q=new RegExp(L+"+","g"),R=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),S=new RegExp("^"+L+"*,"+L+"*"),T=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),U=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),V=new RegExp(P),W=new RegExp("^"+N+"$"),X={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M.replace("w","w*")+")"),ATTR:new RegExp("^"+O),PSEUDO:new RegExp("^"+P),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},eb=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(fb){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function gb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],k=b.nodeType,"string"!=typeof a||!a||1!==k&&9!==k&&11!==k)return d;if(!e&&p){if(11!==k&&(f=_.exec(a)))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return H.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName)return H.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=1!==k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+rb(o[l]);w=ab.test(a)&&pb(b.parentNode)||b,x=o.join(",")}if(x)try{return H.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function hb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ib(a){return a[u]=!0,a}function jb(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function kb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function lb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function nb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function ob(a){return ib(function(b){return b=+b,ib(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function pb(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=gb.support={},f=gb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=gb.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=g.documentElement,e=g.defaultView,e&&e!==e.top&&(e.addEventListener?e.addEventListener("unload",eb,!1):e.attachEvent&&e.attachEvent("onunload",eb)),p=!f(g),c.attributes=jb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=jb(function(a){return a.appendChild(g.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(g.getElementsByClassName),c.getById=jb(function(a){return o.appendChild(a).id=u,!g.getElementsByName||!g.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(g.querySelectorAll))&&(jb(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),jb(function(a){var b=g.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&jb(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",P)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===g||a.ownerDocument===v&&t(v,a)?-1:b===g||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,h=[a],i=[b];if(!e||!f)return a===g?-1:b===g?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return lb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?lb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},g):n},gb.matches=function(a,b){return gb(a,null,null,b)},gb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return gb(b,n,null,[a]).length>0},gb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},gb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},gb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},gb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=gb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=gb.selectors={cacheLength:50,createPseudo:ib,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||gb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&gb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=gb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(Q," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||gb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ib(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ib(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?ib(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ib(function(a){return function(b){return gb(a,b).length>0}}),contains:ib(function(a){return a=a.replace(cb,db),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ib(function(a){return W.test(a||"")||gb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:ob(function(){return[0]}),last:ob(function(a,b){return[b-1]}),eq:ob(function(a,b,c){return[0>c?c+b:c]}),even:ob(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:ob(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:ob(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:ob(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function sb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function tb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ub(a,b,c){for(var d=0,e=b.length;e>d;d++)gb(a,b[d],c);return c}function vb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function wb(a,b,c,d,e,f){return d&&!d[u]&&(d=wb(d)),e&&!e[u]&&(e=wb(e,f)),ib(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ub(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:vb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=vb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=vb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function xb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=sb(function(a){return a===b},h,!0),l=sb(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[sb(tb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return wb(i>1&&tb(m),i>1&&rb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&xb(a.slice(i,e)),f>e&&xb(a=a.slice(e)),f>e&&rb(a))}m.push(c)}return tb(m)}function yb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=F.call(i));s=vb(s)}H.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&gb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?ib(f):f}return h=gb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=xb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,yb(e,d)),f.selector=a}return f},i=gb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&pb(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&rb(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&pb(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=jb(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),jb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||kb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&jb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||kb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),jb(function(a){return null==a.getAttribute("disabled")})||kb(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),gb}(a);m.find=s,m.expr=s.selectors,m.expr[":"]=m.expr.pseudos,m.unique=s.uniqueSort,m.text=s.getText,m.isXMLDoc=s.isXML,m.contains=s.contains;var t=m.expr.match.needsContext,u=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,v=/^.[^:#\[\.,]*$/;function w(a,b,c){if(m.isFunction(b))return m.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return m.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(v.test(b))return m.filter(b,a,c);b=m.filter(b,a)}return m.grep(a,function(a){return m.inArray(a,b)>=0!==c})}m.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?m.find.matchesSelector(d,a)?[d]:[]:m.find.matches(a,m.grep(b,function(a){return 1===a.nodeType}))},m.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(m(a).filter(function(){for(b=0;e>b;b++)if(m.contains(d[b],this))return!0}));for(b=0;e>b;b++)m.find(a,d[b],c);return c=this.pushStack(e>1?m.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(w(this,a||[],!1))},not:function(a){return this.pushStack(w(this,a||[],!0))},is:function(a){return!!w(this,"string"==typeof a&&t.test(a)?m(a):a||[],!1).length}});var x,y=a.document,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=m.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||x).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof m?b[0]:b,m.merge(this,m.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:y,!0)),u.test(c[1])&&m.isPlainObject(b))for(c in b)m.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=y.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return x.find(a);this.length=1,this[0]=d}return this.context=y,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):m.isFunction(a)?"undefined"!=typeof x.ready?x.ready(a):a(m):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),m.makeArray(a,this))};A.prototype=m.fn,x=m(y);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};m.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!m(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),m.fn.extend({has:function(a){var b,c=m(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(m.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=t.test(a)||"string"!=typeof a?m(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&m.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?m.unique(f):f)},index:function(a){return a?"string"==typeof a?m.inArray(this[0],m(a)):m.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(m.unique(m.merge(this.get(),m(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}m.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return m.dir(a,"parentNode")},parentsUntil:function(a,b,c){return m.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return m.dir(a,"nextSibling")},prevAll:function(a){return m.dir(a,"previousSibling")},nextUntil:function(a,b,c){return m.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return m.dir(a,"previousSibling",c)},siblings:function(a){return m.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return m.sibling(a.firstChild)},contents:function(a){return m.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:m.merge([],a.childNodes)}},function(a,b){m.fn[a]=function(c,d){var e=m.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=m.filter(d,e)),this.length>1&&(C[a]||(e=m.unique(e)),B.test(a)&&(e=e.reverse())),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return m.each(a.match(E)||[],function(a,c){b[c]=!0}),b}m.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):m.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){m.each(b,function(b,c){var d=m.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&m.each(arguments,function(a,c){var d;while((d=m.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?m.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},m.extend({Deferred:function(a){var b=[["resolve","done",m.Callbacks("once memory"),"resolved"],["reject","fail",m.Callbacks("once memory"),"rejected"],["notify","progress",m.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return m.Deferred(function(c){m.each(b,function(b,f){var g=m.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&m.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?m.extend(a,d):d}},e={};return d.pipe=d.then,m.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&m.isFunction(a.promise)?e:0,g=1===f?a:m.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&m.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;m.fn.ready=function(a){return m.ready.promise().done(a),this},m.extend({isReady:!1,readyWait:1,holdReady:function(a){a?m.readyWait++:m.ready(!0)},ready:function(a){if(a===!0?!--m.readyWait:!m.isReady){if(!y.body)return setTimeout(m.ready);m.isReady=!0,a!==!0&&--m.readyWait>0||(H.resolveWith(y,[m]),m.fn.triggerHandler&&(m(y).triggerHandler("ready"),m(y).off("ready")))}}});function I(){y.addEventListener?(y.removeEventListener("DOMContentLoaded",J,!1),a.removeEventListener("load",J,!1)):(y.detachEvent("onreadystatechange",J),a.detachEvent("onload",J))}function J(){(y.addEventListener||"load"===event.type||"complete"===y.readyState)&&(I(),m.ready())}m.ready.promise=function(b){if(!H)if(H=m.Deferred(),"complete"===y.readyState)setTimeout(m.ready);else if(y.addEventListener)y.addEventListener("DOMContentLoaded",J,!1),a.addEventListener("load",J,!1);else{y.attachEvent("onreadystatechange",J),a.attachEvent("onload",J);var c=!1;try{c=null==a.frameElement&&y.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!m.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}I(),m.ready()}}()}return H.promise(b)};var K="undefined",L;for(L in m(k))break;k.ownLast="0"!==L,k.inlineBlockNeedsLayout=!1,m(function(){var a,b,c,d;c=y.getElementsByTagName("body")[0],c&&c.style&&(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),typeof b.style.zoom!==K&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",k.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(d))}),function(){var a=y.createElement("div");if(null==k.deleteExpando){k.deleteExpando=!0;try{delete a.test}catch(b){k.deleteExpando=!1}}a=null}(),m.acceptData=function(a){var b=m.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var M=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,N=/([A-Z])/g;function O(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(N,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:M.test(c)?m.parseJSON(c):c}catch(e){}m.data(a,b,c)}else c=void 0}return c}function P(a){var b;for(b in a)if(("data"!==b||!m.isEmptyObject(a[b]))&&"toJSON"!==b)return!1; -return!0}function Q(a,b,d,e){if(m.acceptData(a)){var f,g,h=m.expando,i=a.nodeType,j=i?m.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||m.guid++:h),j[k]||(j[k]=i?{}:{toJSON:m.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=m.extend(j[k],b):j[k].data=m.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[m.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[m.camelCase(b)])):f=g,f}}function R(a,b,c){if(m.acceptData(a)){var d,e,f=a.nodeType,g=f?m.cache:a,h=f?a[m.expando]:m.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){m.isArray(b)?b=b.concat(m.map(b,m.camelCase)):b in d?b=[b]:(b=m.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!P(d):!m.isEmptyObject(d))return}(c||(delete g[h].data,P(g[h])))&&(f?m.cleanData([a],!0):k.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}m.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?m.cache[a[m.expando]]:a[m.expando],!!a&&!P(a)},data:function(a,b,c){return Q(a,b,c)},removeData:function(a,b){return R(a,b)},_data:function(a,b,c){return Q(a,b,c,!0)},_removeData:function(a,b){return R(a,b,!0)}}),m.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=m.data(f),1===f.nodeType&&!m._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=m.camelCase(d.slice(5)),O(f,d,e[d])));m._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){m.data(this,a)}):arguments.length>1?this.each(function(){m.data(this,a,b)}):f?O(f,a,m.data(f,a)):void 0},removeData:function(a){return this.each(function(){m.removeData(this,a)})}}),m.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=m._data(a,b),c&&(!d||m.isArray(c)?d=m._data(a,b,m.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=m.queue(a,b),d=c.length,e=c.shift(),f=m._queueHooks(a,b),g=function(){m.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return m._data(a,c)||m._data(a,c,{empty:m.Callbacks("once memory").add(function(){m._removeData(a,b+"queue"),m._removeData(a,c)})})}}),m.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},W=/^(?:checkbox|radio)$/i;!function(){var a=y.createElement("input"),b=y.createElement("div"),c=y.createDocumentFragment();if(b.innerHTML="
a",k.leadingWhitespace=3===b.firstChild.nodeType,k.tbody=!b.getElementsByTagName("tbody").length,k.htmlSerialize=!!b.getElementsByTagName("link").length,k.html5Clone="<:nav>"!==y.createElement("nav").cloneNode(!0).outerHTML,a.type="checkbox",a.checked=!0,c.appendChild(a),k.appendChecked=a.checked,b.innerHTML="",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,c.appendChild(b),b.innerHTML="",k.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,k.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){k.noCloneEvent=!1}),b.cloneNode(!0).click()),null==k.deleteExpando){k.deleteExpando=!0;try{delete b.test}catch(d){k.deleteExpando=!1}}}(),function(){var b,c,d=y.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(k[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),k[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var X=/^(?:input|select|textarea)$/i,Y=/^key/,Z=/^(?:mouse|pointer|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=/^([^.]*)(?:\.(.+)|)$/;function ab(){return!0}function bb(){return!1}function cb(){try{return y.activeElement}catch(a){}}m.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=m.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof m===K||a&&m.event.triggered===a.type?void 0:m.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(E)||[""],h=b.length;while(h--)f=_.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=m.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=m.event.special[o]||{},l=m.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&m.expr.match.needsContext.test(e),namespace:p.join(".")},i),(n=g[o])||(n=g[o]=[],n.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?n.splice(n.delegateCount++,0,l):n.push(l),m.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m.hasData(a)&&m._data(a);if(r&&(k=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=_.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=m.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,n=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=n.length;while(f--)g=n[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(n.splice(f,1),g.selector&&n.delegateCount--,l.remove&&l.remove.call(a,g));i&&!n.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||m.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)m.event.remove(a,o+b[j],c,d,!0);m.isEmptyObject(k)&&(delete r.handle,m._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,n,o=[d||y],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||y,3!==d.nodeType&&8!==d.nodeType&&!$.test(p+m.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[m.expando]?b:new m.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:m.makeArray(c,[b]),k=m.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!m.isWindow(d)){for(i=k.delegateType||p,$.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||y)&&o.push(l.defaultView||l.parentWindow||a)}n=0;while((h=o[n++])&&!b.isPropagationStopped())b.type=n>1?i:k.bindType||p,f=(m._data(h,"events")||{})[b.type]&&m._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&m.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&m.acceptData(d)&&g&&d[p]&&!m.isWindow(d)){l=d[g],l&&(d[g]=null),m.event.triggered=p;try{d[p]()}catch(r){}m.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=m.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(m._data(this,"events")||{})[a.type]||[],k=m.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=m.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((m.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?m(c,this).index(i)>=0:m.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h]","i"),hb=/^\s+/,ib=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,jb=/<([\w:]+)/,kb=/\s*$/g,rb={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:k.htmlSerialize?[0,"",""]:[1,"X
","
"]},sb=db(y),tb=sb.appendChild(y.createElement("div"));rb.optgroup=rb.option,rb.tbody=rb.tfoot=rb.colgroup=rb.caption=rb.thead,rb.th=rb.td;function ub(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==K?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==K?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||m.nodeName(d,b)?f.push(d):m.merge(f,ub(d,b));return void 0===b||b&&m.nodeName(a,b)?m.merge([a],f):f}function vb(a){W.test(a.type)&&(a.defaultChecked=a.checked)}function wb(a,b){return m.nodeName(a,"table")&&m.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function xb(a){return a.type=(null!==m.find.attr(a,"type"))+"/"+a.type,a}function yb(a){var b=pb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function zb(a,b){for(var c,d=0;null!=(c=a[d]);d++)m._data(c,"globalEval",!b||m._data(b[d],"globalEval"))}function Ab(a,b){if(1===b.nodeType&&m.hasData(a)){var c,d,e,f=m._data(a),g=m._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)m.event.add(b,c,h[c][d])}g.data&&(g.data=m.extend({},g.data))}}function Bb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!k.noCloneEvent&&b[m.expando]){e=m._data(b);for(d in e.events)m.removeEvent(b,d,e.handle);b.removeAttribute(m.expando)}"script"===c&&b.text!==a.text?(xb(b).text=a.text,yb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),k.html5Clone&&a.innerHTML&&!m.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&W.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}m.extend({clone:function(a,b,c){var d,e,f,g,h,i=m.contains(a.ownerDocument,a);if(k.html5Clone||m.isXMLDoc(a)||!gb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(tb.innerHTML=a.outerHTML,tb.removeChild(f=tb.firstChild)),!(k.noCloneEvent&&k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||m.isXMLDoc(a)))for(d=ub(f),h=ub(a),g=0;null!=(e=h[g]);++g)d[g]&&Bb(e,d[g]);if(b)if(c)for(h=h||ub(a),d=d||ub(f),g=0;null!=(e=h[g]);g++)Ab(e,d[g]);else Ab(a,f);return d=ub(f,"script"),d.length>0&&zb(d,!i&&ub(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,l,n=a.length,o=db(b),p=[],q=0;n>q;q++)if(f=a[q],f||0===f)if("object"===m.type(f))m.merge(p,f.nodeType?[f]:f);else if(lb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(jb.exec(f)||["",""])[1].toLowerCase(),l=rb[i]||rb._default,h.innerHTML=l[1]+f.replace(ib,"<$1>")+l[2],e=l[0];while(e--)h=h.lastChild;if(!k.leadingWhitespace&&hb.test(f)&&p.push(b.createTextNode(hb.exec(f)[0])),!k.tbody){f="table"!==i||kb.test(f)?""!==l[1]||kb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)m.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}m.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),k.appendChecked||m.grep(ub(p,"input"),vb),q=0;while(f=p[q++])if((!d||-1===m.inArray(f,d))&&(g=m.contains(f.ownerDocument,f),h=ub(o.appendChild(f),"script"),g&&zb(h),c)){e=0;while(f=h[e++])ob.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=m.expando,j=m.cache,l=k.deleteExpando,n=m.event.special;null!=(d=a[h]);h++)if((b||m.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)n[e]?m.event.remove(d,e):m.removeEvent(d,e,g.handle);j[f]&&(delete j[f],l?delete d[i]:typeof d.removeAttribute!==K?d.removeAttribute(i):d[i]=null,c.push(f))}}}),m.fn.extend({text:function(a){return V(this,function(a){return void 0===a?m.text(this):this.empty().append((this[0]&&this[0].ownerDocument||y).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?m.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||m.cleanData(ub(c)),c.parentNode&&(b&&m.contains(c.ownerDocument,c)&&zb(ub(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&m.cleanData(ub(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&m.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return m.clone(this,a,b)})},html:function(a){return V(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(fb,""):void 0;if(!("string"!=typeof a||mb.test(a)||!k.htmlSerialize&&gb.test(a)||!k.leadingWhitespace&&hb.test(a)||rb[(jb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(ib,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(m.cleanData(ub(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,m.cleanData(ub(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,n=this,o=l-1,p=a[0],q=m.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&nb.test(p))return this.each(function(c){var d=n.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(i=m.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=m.map(ub(i,"script"),xb),f=g.length;l>j;j++)d=i,j!==o&&(d=m.clone(d,!0,!0),f&&m.merge(g,ub(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,m.map(g,yb),j=0;f>j;j++)d=g[j],ob.test(d.type||"")&&!m._data(d,"globalEval")&&m.contains(h,d)&&(d.src?m._evalUrl&&m._evalUrl(d.src):m.globalEval((d.text||d.textContent||d.innerHTML||"").replace(qb,"")));i=c=null}return this}}),m.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){m.fn[a]=function(a){for(var c,d=0,e=[],g=m(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),m(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Cb,Db={};function Eb(b,c){var d,e=m(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:m.css(e[0],"display");return e.detach(),f}function Fb(a){var b=y,c=Db[a];return c||(c=Eb(a,b),"none"!==c&&c||(Cb=(Cb||m("