From 5fc4a36a144c3f8f22be7e21a4e542d3042d10b1 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 13 Jun 2018 13:43:53 +0200 Subject: [PATCH 1/6] [FLINK-9377] [core] (part 1) Extend TypeSerializerConfigSnapshot as a factory for restoring serializers This commit is the first step towards removing serializers from checkpointed state meta info and making Flink checkpoints Java serialization free. Instead of writing serializers in checkpoints, and trying to read that to obtain a restore serializer at restore time, we aim to only write the config snapshot as the single source of truth and use it as a factory to create a restore serializer. This commit adds the method and signatures to the TypeSerializerConfigSnapshot interface. Use of the method, as well as properly implementing the method for all serializers, will be implemented in follow-up commits. --- .../typeutils/runtime/WritableSerializer.java | 4 +- ...CompositeTypeSerializerConfigSnapshot.java | 2 +- .../GenericTypeSerializerConfigSnapshot.java | 2 +- .../ParameterlessTypeSerializerConfig.java | 2 +- .../typeutils/TypeDeserializerAdapter.java | 4 +- .../api/common/typeutils/TypeSerializer.java | 6 +- .../TypeSerializerConfigSnapshot.java | 51 +++++++++++-- .../UnloadableDummyTypeSerializer.java | 4 +- .../CollectionSerializerConfigSnapshot.java | 5 +- .../common/typeutils/base/EnumSerializer.java | 2 +- .../base/GenericArraySerializer.java | 4 +- .../GenericArraySerializerConfigSnapshot.java | 2 +- .../common/typeutils/base/ListSerializer.java | 4 +- .../common/typeutils/base/MapSerializer.java | 6 +- .../base/MapSerializerConfigSnapshot.java | 4 +- .../base/TypeSerializerSingleton.java | 8 +- .../runtime/CopyableValueSerializer.java | 4 +- .../typeutils/runtime/EitherSerializer.java | 6 +- .../EitherSerializerConfigSnapshot.java | 2 +- .../typeutils/runtime/PojoSerializer.java | 2 +- .../java/typeutils/runtime/RowSerializer.java | 4 +- .../runtime/TupleSerializerBase.java | 2 +- .../TupleSerializerConfigSnapshot.java | 2 +- .../typeutils/runtime/ValueSerializer.java | 2 +- .../runtime/kryo/KryoSerializer.java | 2 +- .../common/typeutils/SerializerTestBase.java | 4 +- .../TypeSerializerSerializationUtilTest.java | 16 ++-- .../typeutils/runtime/AvroSerializer.java | 2 +- .../avro/typeutils/AvroSerializer.java | 16 ++-- .../BackwardsCompatibleAvroSerializer.java | 4 +- .../java/io/CollectionInputFormatTest.java | 4 +- .../java/org/apache/flink/cep/nfa/NFA.java | 6 +- .../apache/flink/cep/nfa/SharedBuffer.java | 7 +- .../flink/cep/nfa/sharedbuffer/Lockable.java | 29 ++++--- .../LockableSerializerConfigSnapshot.java | 43 +++++++++++ .../ListViewSerializerConfigSnapshot.java | 43 +++++++++++ .../MapViewSerializerConfigSnapshot.java | 44 +++++++++++ .../table/dataview/ListViewSerializer.scala | 55 ++++++++------ .../table/dataview/MapViewSerializer.scala | 68 +++++++++-------- .../table/runtime/types/CRowSerializer.scala | 6 +- .../runtime/state/ArrayListSerializer.java | 6 +- .../testutils/types/IntListSerializer.java | 4 +- .../testutils/types/IntPairSerializer.java | 4 +- .../testutils/types/StringPairSerializer.java | 4 +- .../runtime/query/KvStateRegistryTest.java | 4 +- .../state/OperatorStateBackendTest.java | 4 +- .../runtime/state/StateBackendTestBase.java | 8 +- .../state/heap/CopyOnWriteStateTableTest.java | 4 +- .../recordutils/RecordSerializer.java | 4 +- .../ScalaEitherSerializerConfigSnapshot.java | 46 +++++++++++ .../ScalaOptionSerializerConfigSnapshot.java | 4 +- .../ScalaTrySerializerConfigSnapshot.java | 4 +- .../TraversableSerializerConfigSnapshot.java | 5 +- .../scala/typeutils/EitherSerializer.scala | 76 +++++++++---------- .../scala/typeutils/EnumValueSerializer.scala | 4 +- .../scala/typeutils/NothingSerializer.scala | 4 +- .../scala/typeutils/OptionSerializer.scala | 6 +- .../typeutils/TraversableSerializer.scala | 8 +- .../api/scala/typeutils/TrySerializer.scala | 6 +- .../api/datastream/CoGroupedStreams.java | 9 ++- .../sink/TwoPhaseCommitSinkFunction.java | 8 +- .../api/operators/TimerHeapInternalTimer.java | 4 +- .../streamrecord/StreamElementSerializer.java | 8 +- 63 files changed, 481 insertions(+), 236 deletions(-) create mode 100644 flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java create mode 100644 flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java diff --git a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java index 161e65b3b0608..cb6c254f9040a 100644 --- a/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java +++ b/flink-connectors/flink-hadoop-compatibility/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableSerializer.java @@ -167,9 +167,9 @@ public WritableSerializerConfigSnapshot snapshotConfiguration() { } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof WritableSerializerConfigSnapshot - && typeClass.equals(((WritableSerializerConfigSnapshot) configSnapshot).getTypeClass())) { + && typeClass.equals(((WritableSerializerConfigSnapshot) configSnapshot).getTypeClass())) { return CompatibilityResult.compatible(); } else { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java index 45b78c1a1f06e..287137c6860a8 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java @@ -38,7 +38,7 @@ * is required. */ @Internal -public abstract class CompositeTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { +public abstract class CompositeTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { private List, TypeSerializerConfigSnapshot>> nestedSerializersAndConfigs; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java index 4edfe12324134..ae95bfdda0c2f 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericTypeSerializerConfigSnapshot.java @@ -31,7 +31,7 @@ * @param The type to be instantiated. */ @Internal -public abstract class GenericTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { +public abstract class GenericTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { private Class typeClass; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java index 7ba7dd452d609..6fc6d17249172 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/ParameterlessTypeSerializerConfig.java @@ -29,7 +29,7 @@ * A base class for {@link TypeSerializerConfigSnapshot}s that do not have any parameters. */ @Internal -public final class ParameterlessTypeSerializerConfig extends TypeSerializerConfigSnapshot { +public final class ParameterlessTypeSerializerConfig extends TypeSerializerConfigSnapshot { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java index fb59602c5f541..72b25146cf2df 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeDeserializerAdapter.java @@ -130,12 +130,12 @@ public void copy(DataInputView source, DataOutputView target) throws IOException "This is a TypeDeserializerAdapter used only for deserialization; this method should not be used."); } - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException( "This is a TypeDeserializerAdapter used only for deserialization; this method should not be used."); } - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException( "This is a TypeDeserializerAdapter used only for deserialization; this method should not be used."); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java index a606a181d1d8f..017e09ebb3811 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java @@ -179,14 +179,14 @@ public abstract class TypeSerializer implements Serializable { * * @return snapshot of the serializer's current configuration (cannot be {@code null}). */ - public abstract TypeSerializerConfigSnapshot snapshotConfiguration(); + public abstract TypeSerializerConfigSnapshot snapshotConfiguration(); /** * Ensure compatibility of this serializer with a preceding serializer that was registered for serialization of * the same managed state (if any - this method is only relevant if this serializer is registered for * serialization of managed state). * - * The compatibility check in this method should be performed by inspecting the preceding serializer's configuration + *

The compatibility check in this method should be performed by inspecting the preceding serializer's configuration * snapshot. The method may reconfigure the serializer (if required and possible) so that it may be compatible, * or provide a signaling result that informs Flink that state migration is necessary before continuing to use * this serializer. @@ -215,5 +215,5 @@ public abstract class TypeSerializer implements Serializable { * * @return the determined compatibility result (cannot be {@code null}). */ - public abstract CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot); + public abstract CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java index 389d141fff0e8..13c8623824127 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshot.java @@ -25,12 +25,33 @@ /** * A {@code TypeSerializerConfigSnapshot} is a point-in-time view of a {@link TypeSerializer's} configuration. - * The configuration snapshot of a serializer is persisted along with checkpoints of the managed state that the - * serializer is registered to. + * The configuration snapshot of a serializer is persisted within checkpoints + * as a single source of meta information about the schema of serialized data in the checkpoint. + * This serves three purposes: * - *

The persisted configuration may later on be used by new serializers to ensure serialization compatibility - * for the same managed state. In order for new serializers to be able to ensure this, the configuration snapshot - * should encode sufficient information about: + *

    + *
  • Capturing serializer parameters and schema: a serializer's configuration snapshot + * represents information about the parameters, state, and schema of a serializer. + * This is explained in more detail below.
    • + * + *
  • Compatibility checks for new serializers: when new serializers are available, + * they need to be checked whether or not they are compatible to read the data written by the previous serializer. + * This is performed by providing the serializer configuration snapshots in checkpoints to the corresponding + * new serializers.
    • + * + *
  • Factory for a read serializer when schema conversion is required: in the case that new + * serializers are not compatible to read previous data, a schema conversion process executed across all data + * is required before the new serializer can be continued to be used. This conversion process requires a compatible + * read serializer to restore serialized bytes as objects, and then written back again using the new serializer. + * In this scenario, the serializer configuration snapshots in checkpoints doubles as a factory for the read + * serializer of the conversion process.
    • + *
+ * + *

Serializer Configuration and Schema

+ * + *

Since serializer configuration snapshots needs to be used to ensure serialization compatibility + * for the same managed state as well as serving as a factory for compatible read serializers, the configuration + * snapshot should encode sufficient information about: * *

    *
  • Parameter settings of the serializer: parameters of the serializer include settings @@ -38,18 +59,34 @@ * has nested serializers, then the configuration snapshot should also contain the parameters of the nested * serializers.
    • * - *
  • Serialization schema of the serializer: the data format used by the serializer.
    • + *
  • Serialization schema of the serializer: the binary format used by the serializer, or + * in other words, the schema of data written by the serializer.
    • *
* *

NOTE: Implementations must contain the default empty nullary constructor. This is required to be able to * deserialize the configuration snapshot from its binary form. + * + * @param The data type that the originating serializer of this configuration serializes. */ @PublicEvolving -public abstract class TypeSerializerConfigSnapshot extends VersionedIOReadableWritable { +public abstract class TypeSerializerConfigSnapshot extends VersionedIOReadableWritable { /** The user code class loader; only relevant if this configuration instance was deserialized from binary form. */ private ClassLoader userCodeClassLoader; + /** + * Creates a serializer using this configuration, that is capable of reading data + * written by the serializer described by this configuration. + * + * @return the restored serializer. + */ + public TypeSerializer restoreSerializer() { + // TODO this method actually should not have a default implementation; + // TODO this placeholder should be removed as soon as all subclasses have a proper implementation in place, and + // TODO the method is properly integrated in state backends' restore procedures + throw new UnsupportedOperationException(); + } + /** * Set the user code class loader. * Only relevant if this configuration instance was deserialized from binary form. diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/UnloadableDummyTypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/UnloadableDummyTypeSerializer.java index ddfeab41c1d37..448c53b209ed4 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/UnloadableDummyTypeSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/UnloadableDummyTypeSerializer.java @@ -94,12 +94,12 @@ public void copy(DataInputView source, DataOutputView target) throws IOException } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException("This object is a dummy TypeSerializer."); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException("This object is a dummy TypeSerializer."); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java index 55729852ed129..d1a3e9508ac60 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java @@ -22,13 +22,16 @@ import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; +import java.util.Collection; + /** * Configuration snapshot of a serializer for collection types. * * @param Type of the element. */ @Internal -public final class CollectionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class CollectionSerializerConfigSnapshot, T> + extends CompositeTypeSerializerConfigSnapshot { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java index c40fefc1694fe..dcd26734f240a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/EnumSerializer.java @@ -178,7 +178,7 @@ public EnumSerializerConfigSnapshot snapshotConfiguration() { @SuppressWarnings("unchecked") @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof EnumSerializerConfigSnapshot) { final EnumSerializerConfigSnapshot config = (EnumSerializerConfigSnapshot) configSnapshot; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java index cdfc964e15d6d..a42226cd91146 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java @@ -198,12 +198,12 @@ public String toString() { // -------------------------------------------------------------------------------------------- @Override - public GenericArraySerializerConfigSnapshot snapshotConfiguration() { + public GenericArraySerializerConfigSnapshot snapshotConfiguration() { return new GenericArraySerializerConfigSnapshot<>(componentClass, componentSerializer); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof GenericArraySerializerConfigSnapshot) { final GenericArraySerializerConfigSnapshot config = (GenericArraySerializerConfigSnapshot) configSnapshot; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java index 70e52106b48d6..5b207ddbb2586 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java @@ -36,7 +36,7 @@ * @param The component type. */ @Internal -public final class GenericArraySerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class GenericArraySerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java index c2b935c82e1eb..b5759740c043a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java @@ -179,7 +179,7 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- @Override - public CollectionSerializerConfigSnapshot snapshotConfiguration() { + public CollectionSerializerConfigSnapshot, T> snapshotConfiguration() { return new CollectionSerializerConfigSnapshot<>(elementSerializer); } @@ -187,7 +187,7 @@ public CollectionSerializerConfigSnapshot snapshotConfiguration() { public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof CollectionSerializerConfigSnapshot) { Tuple2, TypeSerializerConfigSnapshot> previousElemSerializerAndConfig = - ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); + ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( previousElemSerializerAndConfig.f0, diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java index 6471152b0bc53..570f8f83a2f74 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java @@ -207,15 +207,15 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- @Override - public MapSerializerConfigSnapshot snapshotConfiguration() { + public MapSerializerConfigSnapshot snapshotConfiguration() { return new MapSerializerConfigSnapshot<>(keySerializer, valueSerializer); } @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof MapSerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> previousKvSerializersAndConfigs = - ((MapSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); + ((MapSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( previousKvSerializersAndConfigs.get(0).f0, diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java index 9db3019ad46c5..e601d4d52cca3 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java @@ -22,12 +22,14 @@ import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; +import java.util.Map; + /** * Configuration snapshot for serializers of maps, containing the * configuration snapshot of its key serializer and value serializer. */ @Internal -public final class MapSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class MapSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java index 9354af07e18a4..13e41675859f7 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/TypeSerializerSingleton.java @@ -53,16 +53,16 @@ public boolean equals(Object obj) { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { // type serializer singletons should always be parameter-less - return new ParameterlessTypeSerializerConfig(getSerializationFormatIdentifier()); + return new ParameterlessTypeSerializerConfig<>(getSerializationFormatIdentifier()); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof ParameterlessTypeSerializerConfig && isCompatibleSerializationFormatIdentifier( - ((ParameterlessTypeSerializerConfig) configSnapshot).getSerializationFormatIdentifier())) { + ((ParameterlessTypeSerializerConfig) configSnapshot).getSerializationFormatIdentifier())) { return CompatibilityResult.compatible(); } else { diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java index b9039690f9ff3..68e4af3ea446d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueSerializer.java @@ -142,9 +142,9 @@ public CopyableValueSerializerConfigSnapshot snapshotConfiguration() { } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof CopyableValueSerializerConfigSnapshot - && valueClass.equals(((CopyableValueSerializerConfigSnapshot) configSnapshot).getTypeClass())) { + && valueClass.equals(((CopyableValueSerializerConfigSnapshot) configSnapshot).getTypeClass())) { return CompatibilityResult.compatible(); } else { return CompatibilityResult.requiresMigration(); diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java index 18ebcd8b63cfd..4f27589e41327 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java @@ -196,15 +196,15 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- @Override - public EitherSerializerConfigSnapshot snapshotConfiguration() { + public EitherSerializerConfigSnapshot snapshotConfiguration() { return new EitherSerializerConfigSnapshot<>(leftSerializer, rightSerializer); } @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof EitherSerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> previousLeftRightSerializersAndConfigs = - ((EitherSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); + ((EitherSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( previousLeftRightSerializersAndConfigs.get(0).f0, diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java index f9968781a8cb6..628fff4110778 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java @@ -28,7 +28,7 @@ * containing configuration snapshots of the Left and Right serializers. */ @Internal -public final class EitherSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class EitherSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java index a4d086d6a7464..cdff1fa0aeca0 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java @@ -594,7 +594,7 @@ public PojoSerializerConfigSnapshot snapshotConfiguration() { @SuppressWarnings("unchecked") @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof PojoSerializerConfigSnapshot) { final PojoSerializerConfigSnapshot config = (PojoSerializerConfigSnapshot) configSnapshot; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java index 7f9cc2145be66..b806c27a97ea9 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java @@ -268,7 +268,7 @@ public RowSerializerConfigSnapshot snapshotConfiguration() { } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof RowSerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> previousFieldSerializersAndConfigs = ((RowSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); @@ -312,7 +312,7 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot return CompatibilityResult.requiresMigration(); } - public static final class RowSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { + public static final class RowSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java index 3fb7defc2c96c..34146ca8e022a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java @@ -135,7 +135,7 @@ public TupleSerializerConfigSnapshot snapshotConfiguration() { @SuppressWarnings("unchecked") @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof TupleSerializerConfigSnapshot) { final TupleSerializerConfigSnapshot config = (TupleSerializerConfigSnapshot) configSnapshot; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java index eac5200da9c6f..a1180aeba5325 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java @@ -32,7 +32,7 @@ * Snapshot of a tuple serializer's configuration. */ @Internal -public final class TupleSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class TupleSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { private static final int VERSION = 1; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java index 0a028ebd79d66..9e02c35901430 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueSerializer.java @@ -182,7 +182,7 @@ public ValueSerializerConfigSnapshot snapshotConfiguration() { @SuppressWarnings("unchecked") @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof ValueSerializerConfigSnapshot) { final ValueSerializerConfigSnapshot config = (ValueSerializerConfigSnapshot) configSnapshot; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java index d8158aabb92db..5ac914e03ef74 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java @@ -463,7 +463,7 @@ public KryoSerializerConfigSnapshot snapshotConfiguration() { @SuppressWarnings("unchecked") @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof KryoSerializerConfigSnapshot) { final KryoSerializerConfigSnapshot config = (KryoSerializerConfigSnapshot) configSnapshot; diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java index 57015c78be0af..d32cb0cd34174 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java @@ -124,7 +124,7 @@ public void testSnapshotConfigurationAndReconfigure() throws Exception { assertFalse(strategy.isRequiresMigration()); // also verify that the serializer's reconfigure implementation detects incompatibility - strategy = getSerializer().ensureCompatibility(new TestIncompatibleSerializerConfigSnapshot()); + strategy = getSerializer().ensureCompatibility(new TestIncompatibleSerializerConfigSnapshot<>()); assertTrue(strategy.isRequiresMigration()); } @@ -526,7 +526,7 @@ public void skipBytesToRead(int numBytes) throws IOException { } } - public static final class TestIncompatibleSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { + public static final class TestIncompatibleSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { @Override public int getVersion() { return 0; diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java index 04739314dccca..a7136fcd18bd2 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java @@ -148,11 +148,11 @@ public void testSerializerSerializationWithInvalidClass() throws Exception { */ @Test public void testSerializeConfigurationSnapshots() throws Exception { - TypeSerializerSerializationUtilTest.TestConfigSnapshot configSnapshot1 = - new TypeSerializerSerializationUtilTest.TestConfigSnapshot(1, "foo"); + TypeSerializerSerializationUtilTest.TestConfigSnapshot configSnapshot1 = + new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(1, "foo"); - TypeSerializerSerializationUtilTest.TestConfigSnapshot configSnapshot2 = - new TypeSerializerSerializationUtilTest.TestConfigSnapshot(2, "bar"); + TypeSerializerSerializationUtilTest.TestConfigSnapshot configSnapshot2 = + new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(2, "bar"); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { @@ -183,7 +183,7 @@ public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception { byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSerializationUtil.writeSerializerConfigSnapshot( - new DataOutputViewStreamWrapper(out), new TypeSerializerSerializationUtilTest.TestConfigSnapshot(123, "foobar")); + new DataOutputViewStreamWrapper(out), new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar")); serializedConfig = out.toByteArray(); } @@ -266,7 +266,7 @@ public void testAnonymousSerializerClassWithChangedSerialVersionUID() throws Exc Assert.assertTrue(anonymousClassSerializer.getClass().isAnonymousClass()); } - public static class TestConfigSnapshot extends TypeSerializerConfigSnapshot { + public static class TestConfigSnapshot extends TypeSerializerConfigSnapshot { static final int VERSION = 1; @@ -411,12 +411,12 @@ public TypeSerializer duplicate() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { return IntSerializer.INSTANCE.snapshotConfiguration(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { return IntSerializer.INSTANCE.ensureCompatibility(configSnapshot); } diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java index f3801991be6ae..5f76b094361e6 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/api/java/typeutils/runtime/AvroSerializer.java @@ -225,7 +225,7 @@ public AvroSerializerConfigSnapshot snapshotConfiguration() { @SuppressWarnings("unchecked") @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof AvroSerializerConfigSnapshot) { final AvroSerializerConfigSnapshot config = (AvroSerializerConfigSnapshot) configSnapshot; diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java index b313625bfe23d..dad1d6df16adf 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java @@ -100,7 +100,7 @@ public class AvroSerializer extends TypeSerializer { private transient Schema schema; /** The serializer configuration snapshot, cached for efficiency. */ - private transient AvroSchemaSerializerConfigSnapshot configSnapshot; + private transient AvroSchemaSerializerConfigSnapshot configSnapshot; /** The currently accessing thread, set and checked on debug level only. */ private transient volatile Thread currentThread; @@ -264,20 +264,20 @@ public void copy(DataInputView source, DataOutputView target) throws IOException // ------------------------------------------------------------------------ @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { if (configSnapshot == null) { checkAvroInitialized(); - configSnapshot = new AvroSchemaSerializerConfigSnapshot(schema.toString(false)); + configSnapshot = new AvroSchemaSerializerConfigSnapshot<>(schema.toString(false)); } return configSnapshot; } @Override - @SuppressWarnings("deprecation") - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + @SuppressWarnings({"deprecation", "unchecked"}) + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof AvroSchemaSerializerConfigSnapshot) { // proper schema snapshot, can do the sophisticated schema-based compatibility check - final String schemaString = ((AvroSchemaSerializerConfigSnapshot) configSnapshot).getSchemaString(); + final String schemaString = ((AvroSchemaSerializerConfigSnapshot) configSnapshot).getSchemaString(); final Schema lastSchema = new Schema.Parser().parse(schemaString); checkAvroInitialized(); @@ -291,7 +291,7 @@ else if (configSnapshot instanceof AvroSerializerConfigSnapshot) { // old snapshot case, just compare the type // we don't need to restore any Kryo stuff, since Kryo was never used for persistence, // only for object-to-object copies. - final AvroSerializerConfigSnapshot old = (AvroSerializerConfigSnapshot) configSnapshot; + final AvroSerializerConfigSnapshot old = (AvroSerializerConfigSnapshot) configSnapshot; return type.equals(old.getTypeClass()) ? CompatibilityResult.compatible() : CompatibilityResult.requiresMigration(); } @@ -419,7 +419,7 @@ private void exitExclusiveThread() { /** * A config snapshot for the Avro Serializer that stores the Avro Schema to check compatibility. */ - public static final class AvroSchemaSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { + public static final class AvroSchemaSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { private String schemaString; diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java index e5eb5d89a1f77..addd4b1295423 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java @@ -172,13 +172,13 @@ public String toString() { // ------------------------------------------------------------------------ @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { // we return the configuration of the actually used serializer here return serializer.snapshotConfiguration(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof AvroSchemaSerializerConfigSnapshot || configSnapshot instanceof AvroSerializerConfigSnapshot) { diff --git a/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java index 579e761c014ff..7b1441729b167 100644 --- a/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java +++ b/flink-java/src/test/java/org/apache/flink/api/java/io/CollectionInputFormatTest.java @@ -395,12 +395,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException(); } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java index 041a01702f472..a6ae7746812fa 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java @@ -863,7 +863,7 @@ public Queue getComputationStates() { * The {@link TypeSerializerConfigSnapshot} serializer configuration to be stored with the managed state. */ @Deprecated - public static final class NFASerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { + public static final class NFASerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; @@ -987,7 +987,7 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot> snapshotConfiguration() { return new NFASerializerConfigSnapshot<>(eventSerializer, sharedBufferSerializer); } @@ -995,7 +995,7 @@ public TypeSerializerConfigSnapshot snapshotConfiguration() { public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof NFASerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> serializersAndConfigs = - ((NFASerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); + ((NFASerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult eventCompatResult = CompatibilityUtil.resolveCompatibilityResult( serializersAndConfigs.get(0).f0, diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java index a4dbc00704da9..4ab227961b0a4 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java @@ -149,7 +149,8 @@ public static ValueTimeWrapper deserialize( /** * The {@link TypeSerializerConfigSnapshot} serializer configuration to be stored with the managed state. */ - public static final class SharedBufferSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { + public static final class SharedBufferSerializerConfigSnapshot + extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; @@ -342,7 +343,7 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot> snapshotConfiguration() { return new SharedBufferSerializerConfigSnapshot<>( keySerializer, valueSerializer, @@ -353,7 +354,7 @@ public TypeSerializerConfigSnapshot snapshotConfiguration() { public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof SharedBufferSerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> serializerConfigSnapshots = - ((SharedBufferSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); + ((SharedBufferSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( serializerConfigSnapshots.get(0).f0, diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java index b782d8a07c6e7..40b015639731e 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java @@ -19,10 +19,10 @@ package org.apache.flink.cep.nfa.sharedbuffer; import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -181,19 +181,28 @@ public boolean canEqual(Object obj) { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { - return elementSerializer.snapshotConfiguration(); + public TypeSerializerConfigSnapshot> snapshotConfiguration() { + return new LockableSerializerConfigSnapshot<>(elementSerializer); } @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - CompatibilityResult inputComaptibilityResult = elementSerializer.ensureCompatibility(configSnapshot); - if (inputComaptibilityResult.isRequiresMigration()) { - return CompatibilityResult.requiresMigration(new LockableTypeSerializer<>( - new TypeDeserializerAdapter<>(inputComaptibilityResult.getConvertDeserializer())) - ); + public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + if (configSnapshot instanceof LockableSerializerConfigSnapshot) { + @SuppressWarnings("unchecked") + LockableSerializerConfigSnapshot snapshot = (LockableSerializerConfigSnapshot) configSnapshot; + + Tuple2, TypeSerializerConfigSnapshot> nestedSerializerAndConfig = + snapshot.getSingleNestedSerializerAndConfig(); + + CompatibilityResult inputComaptibilityResult = elementSerializer + .internalEnsureCompatibility(nestedSerializerAndConfig.f1); + if (inputComaptibilityResult.isRequiresMigration()) { + return CompatibilityResult.requiresMigration(); + } else { + return CompatibilityResult.compatible(); + } } else { - return CompatibilityResult.compatible(); + return CompatibilityResult.requiresMigration(); } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..f3702ac0459a7 --- /dev/null +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cep.nfa.sharedbuffer; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; + +/** + * A {@link TypeSerializerConfigSnapshot} for the {@link Lockable.LockableTypeSerializer}. + */ +@Internal +public class LockableSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { + + private static final int VERSION = 1; + + public LockableSerializerConfigSnapshot(TypeSerializer elementSerializer) { + super(elementSerializer); + } + + @Override + public int getVersion() { + return VERSION; + } + +} diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..6230e811c4658 --- /dev/null +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.dataview; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.base.ListSerializer; +import org.apache.flink.table.api.dataview.ListView; + +/** + * A {@link TypeSerializerConfigSnapshot} for the {@link ListViewSerializer}. + * + * @param the type of the list elements. + */ +public final class ListViewSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { + + private static final int VERSION = 1; + + public ListViewSerializerConfigSnapshot(ListSerializer listSerializer) { + super(listSerializer); + } + + @Override + public int getVersion() { + return VERSION; + } +} diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..36954b95b236a --- /dev/null +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.dataview; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.base.MapSerializer; +import org.apache.flink.table.api.dataview.MapView; + +/** + * A {@link TypeSerializerConfigSnapshot} for the {@link MapViewSerializer}. + * + * @param the key type of the map entries. + * @param the value type of the map entries. + */ +public class MapViewSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { + + private static final int VERSION = 1; + + public MapViewSerializerConfigSnapshot(MapSerializer mapSerializer) { + super(mapSerializer); + } + + @Override + public int getVersion() { + return VERSION; + } +} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala index a450c2ce1e552..71ef183943f27 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala @@ -75,37 +75,42 @@ class ListViewSerializer[T](val listSerializer: ListSerializer[T]) override def equals(obj: Any): Boolean = canEqual(this) && listSerializer.equals(obj.asInstanceOf[ListViewSerializer[_]].listSerializer) - override def snapshotConfiguration(): TypeSerializerConfigSnapshot = - listSerializer.snapshotConfiguration() + override def snapshotConfiguration(): ListViewSerializerConfigSnapshot[T] = + new ListViewSerializerConfigSnapshot[T](listSerializer) - // copy and modified from ListSerializer.ensureCompatibility override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[ListView[T]] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[ListView[T]] = { configSnapshot match { - case snapshot: CollectionSerializerConfigSnapshot[_] => - val previousListSerializerAndConfig = snapshot.getSingleNestedSerializerAndConfig - - val compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousListSerializerAndConfig.f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousListSerializerAndConfig.f1, - listSerializer.getElementSerializer) - - if (!compatResult.isRequiresMigration) { - CompatibilityResult.compatible[ListView[T]] - } else if (compatResult.getConvertDeserializer != null) { - CompatibilityResult.requiresMigration( - new ListViewSerializer[T]( - new ListSerializer[T]( - new TypeDeserializerAdapter[T](compatResult.getConvertDeserializer)) - ) - ) - } else { - CompatibilityResult.requiresMigration[ListView[T]] - } + case snapshot: ListViewSerializerConfigSnapshot[T] => + checkCompatibility(snapshot) + + // backwards compatibility path; + // Flink versions older or equal to 1.5.x returns a + // CollectionSerializerConfigSnapshot as the snapshot + case legacySnapshot: CollectionSerializerConfigSnapshot[java.util.List[T], T] => + checkCompatibility(legacySnapshot) case _ => CompatibilityResult.requiresMigration[ListView[T]] } } + + private def checkCompatibility( + configSnapshot: CompositeTypeSerializerConfigSnapshot[_] + ): CompatibilityResult[ListView[T]] = { + + val previousListSerializerAndConfig = configSnapshot.getSingleNestedSerializerAndConfig + + val compatResult = CompatibilityUtil.resolveCompatibilityResult( + previousListSerializerAndConfig.f0, + classOf[UnloadableDummyTypeSerializer[_]], + previousListSerializerAndConfig.f1, + listSerializer.getElementSerializer) + + if (!compatResult.isRequiresMigration) { + CompatibilityResult.compatible[ListView[T]] + } else { + CompatibilityResult.requiresMigration[ListView[T]] + } + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala index c53f10c37e597..0a98e34ede0d2 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala @@ -77,45 +77,49 @@ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) override def equals(obj: Any): Boolean = canEqual(this) && mapSerializer.equals(obj.asInstanceOf[MapViewSerializer[_, _]].mapSerializer) - override def snapshotConfiguration(): TypeSerializerConfigSnapshot = - mapSerializer.snapshotConfiguration() + override def snapshotConfiguration(): MapViewSerializerConfigSnapshot[K, V] = + new MapViewSerializerConfigSnapshot[K, V](mapSerializer) // copy and modified from MapSerializer.ensureCompatibility - override def ensureCompatibility(configSnapshot: TypeSerializerConfigSnapshot) + override def ensureCompatibility(configSnapshot: TypeSerializerConfigSnapshot[_]) : CompatibilityResult[MapView[K, V]] = { configSnapshot match { - case snapshot: MapSerializerConfigSnapshot[_, _] => - val previousKvSerializersAndConfigs = snapshot.getNestedSerializersAndConfigs - - val keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(0).f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousKvSerializersAndConfigs.get(0).f1, - mapSerializer.getKeySerializer) - - val valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(1).f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousKvSerializersAndConfigs.get(1).f1, - mapSerializer.getValueSerializer) - - if (!keyCompatResult.isRequiresMigration && !valueCompatResult.isRequiresMigration) { - CompatibilityResult.compatible[MapView[K, V]] - } else if (keyCompatResult.getConvertDeserializer != null - && valueCompatResult.getConvertDeserializer != null) { - CompatibilityResult.requiresMigration( - new MapViewSerializer[K, V]( - new MapSerializer[K, V]( - new TypeDeserializerAdapter[K](keyCompatResult.getConvertDeserializer), - new TypeDeserializerAdapter[V](valueCompatResult.getConvertDeserializer)) - ) - ) - } else { - CompatibilityResult.requiresMigration[MapView[K, V]] - } + case snapshot: MapViewSerializerConfigSnapshot[K, V] => + checkCompatibility(snapshot) + + // backwards compatibility path; + // Flink versions older or equal to 1.5.x returns a + // MapSerializerConfigSnapshot as the snapshot + case legacySnapshot: MapSerializerConfigSnapshot[K, V] => + checkCompatibility(legacySnapshot) case _ => CompatibilityResult.requiresMigration[MapView[K, V]] } } + + private def checkCompatibility( + configSnapshot: CompositeTypeSerializerConfigSnapshot[_] + ): CompatibilityResult[MapView[K, V]] = { + + val previousKvSerializersAndConfigs = configSnapshot.getNestedSerializersAndConfigs + + val keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( + previousKvSerializersAndConfigs.get(0).f0, + classOf[UnloadableDummyTypeSerializer[_]], + previousKvSerializersAndConfigs.get(0).f1, + mapSerializer.getKeySerializer) + + val valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( + previousKvSerializersAndConfigs.get(1).f0, + classOf[UnloadableDummyTypeSerializer[_]], + previousKvSerializersAndConfigs.get(1).f1, + mapSerializer.getValueSerializer) + + if (!keyCompatResult.isRequiresMigration && !valueCompatResult.isRequiresMigration) { + CompatibilityResult.compatible[MapView[K, V]] + } else { + CompatibilityResult.requiresMigration[MapView[K, V]] + } + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala index caf346c3430bd..e96da82dc3bb9 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala @@ -80,12 +80,12 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - override def snapshotConfiguration(): TypeSerializerConfigSnapshot = { + override def snapshotConfiguration(): TypeSerializerConfigSnapshot[CRow] = { new CRowSerializer.CRowSerializerConfigSnapshot(rowSerializer) } override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[CRow] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[CRow] = { configSnapshot match { case crowSerializerConfigSnapshot: CRowSerializer.CRowSerializerConfigSnapshot => @@ -117,7 +117,7 @@ object CRowSerializer { class CRowSerializerConfigSnapshot( private val rowSerializer: TypeSerializer[Row]) - extends CompositeTypeSerializerConfigSnapshot(rowSerializer) { + extends CompositeTypeSerializerConfigSnapshot[CRow](rowSerializer) { /** This empty nullary constructor is required for deserializing the configuration. */ def this() = this(null) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java index 3c4f4b0a26949..1868abf311c16 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java @@ -145,15 +145,15 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot> snapshotConfiguration() { return new CollectionSerializerConfigSnapshot<>(elementSerializer); } @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof CollectionSerializerConfigSnapshot) { Tuple2, TypeSerializerConfigSnapshot> previousElemSerializerAndConfig = - ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); + ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( previousElemSerializerAndConfig.f0, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java index 4bdc5e85cdd31..67d8a26c81357 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListSerializer.java @@ -131,12 +131,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java index 0ae5e71d42803..fe456c8204cfc 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairSerializer.java @@ -140,12 +140,12 @@ public boolean equals(Object obj) { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java index 17ee5f1530b4a..cad78e7c01817 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairSerializer.java @@ -108,12 +108,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/query/KvStateRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/query/KvStateRegistryTest.java index c1c56bf250b87..82d89e05a4dcd 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/query/KvStateRegistryTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/query/KvStateRegistryTest.java @@ -401,12 +401,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { return null; } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { return null; } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java index d8918e784787b..bbf7fd5676382 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java @@ -351,12 +351,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { return IntSerializer.INSTANCE.snapshotConfiguration(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { return IntSerializer.INSTANCE.ensureCompatibility(configSnapshot); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java index ad67171a01c87..34ba80ee2319a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java @@ -4242,14 +4242,14 @@ private TestReconfigurableCustomTypeSerializer(boolean reconfigured) { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { - return new ParameterlessTypeSerializerConfig(getClass().getName()); + public TypeSerializerConfigSnapshot snapshotConfiguration() { + return new ParameterlessTypeSerializerConfig<>(getClass().getName()); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof ParameterlessTypeSerializerConfig && - ((ParameterlessTypeSerializerConfig) configSnapshot).getSerializationFormatIdentifier().equals(getClass().getName())) { + ((ParameterlessTypeSerializerConfig) configSnapshot).getSerializationFormatIdentifier().equals(getClass().getName())) { this.reconfigured = true; return CompatibilityResult.compatible(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java index 4f36d62592249..dbccec1a3fddf 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/heap/CopyOnWriteStateTableTest.java @@ -654,12 +654,12 @@ public void disable() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java index ce23f30006be4..87d91e2ea0ea8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/recordutils/RecordSerializer.java @@ -145,12 +145,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot snapshotConfiguration() { throw new UnsupportedOperationException(); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException(); } } diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..9566a663169b0 --- /dev/null +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.scala.typeutils; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +import scala.util.Either; + +/** + * Configuration snapshot for serializers of Scala's {@link Either} type, + * containing configuration snapshots of the Left and Right serializers. + */ +public class ScalaEitherSerializerConfigSnapshot, L, R> + extends CompositeTypeSerializerConfigSnapshot { + + private static final int VERSION = 1; + + /** This empty nullary constructor is required for deserializing the configuration. */ + public ScalaEitherSerializerConfigSnapshot() {} + + public ScalaEitherSerializerConfigSnapshot(TypeSerializer leftSerializer, TypeSerializer rightSerializer) { + super(leftSerializer, rightSerializer); + } + + @Override + public int getVersion() { + return VERSION; + } +} diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java index 03eef12e9ca4a..215bd447c05d4 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java @@ -22,6 +22,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import scala.Option; + /** * A {@link TypeSerializerConfigSnapshot} for the Scala {@link OptionSerializer}. * @@ -29,7 +31,7 @@ * allow calling different base class constructors from subclasses, while we need that * for the default empty constructor. */ -public final class ScalaOptionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class ScalaOptionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java index 6abb3ea06d761..72baca44193bb 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java @@ -22,6 +22,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import scala.util.Try; + /** * A {@link TypeSerializerConfigSnapshot} for the Scala {@link TrySerializer}. * @@ -29,7 +31,7 @@ * allow calling different base class constructors from subclasses, while we need that * for the default empty constructor. */ -public class ScalaTrySerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public class ScalaTrySerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java index 9a39421dad58a..b7f9ca6ec1afb 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java @@ -22,6 +22,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import scala.collection.TraversableOnce; + /** * A {@link TypeSerializerConfigSnapshot} for the Scala {@link TraversableSerializer}. * @@ -29,7 +31,8 @@ * allow calling different base class constructors from subclasses, while we need that * for the default empty constructor. */ -public class TraversableSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public class TraversableSerializerConfigSnapshot, E> + extends CompositeTypeSerializerConfigSnapshot { private static final int VERSION = 1; diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala index 439e0c2865d16..82637befdcd1a 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala @@ -111,51 +111,51 @@ class EitherSerializer[A, B, T <: Either[A, B]]( // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - override def snapshotConfiguration(): EitherSerializerConfigSnapshot[A, B] = { - new EitherSerializerConfigSnapshot[A, B](leftSerializer, rightSerializer) + override def snapshotConfiguration(): ScalaEitherSerializerConfigSnapshot[T, A, B] = { + new ScalaEitherSerializerConfigSnapshot[T, A, B](leftSerializer, rightSerializer) } override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[T] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[T] = { configSnapshot match { - case eitherSerializerConfig: EitherSerializerConfigSnapshot[A, B] => - val previousLeftRightSerWithConfigs = - eitherSerializerConfig.getNestedSerializersAndConfigs - - val leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerWithConfigs.get(0).f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousLeftRightSerWithConfigs.get(0).f1, - leftSerializer) - - val rightCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerWithConfigs.get(1).f0, - classOf[UnloadableDummyTypeSerializer[_]], - previousLeftRightSerWithConfigs.get(1).f1, - rightSerializer) - - if (leftCompatResult.isRequiresMigration - || rightCompatResult.isRequiresMigration) { - - if (leftCompatResult.getConvertDeserializer != null - && rightCompatResult.getConvertDeserializer != null) { - - CompatibilityResult.requiresMigration( - new EitherSerializer[A, B, T]( - new TypeDeserializerAdapter(leftCompatResult.getConvertDeserializer), - new TypeDeserializerAdapter(rightCompatResult.getConvertDeserializer) - ) - ) - - } else { - CompatibilityResult.requiresMigration() - } - } else { - CompatibilityResult.compatible() - } + case eitherSerializerConfig: ScalaEitherSerializerConfigSnapshot[T, A, B] => + checkCompatibility(eitherSerializerConfig) + + // backwards compatibility path; + // Flink versions older or equal to 1.5.x uses a + // EitherSerializerConfigSnapshot as the snapshot + case legacyConfig: EitherSerializerConfigSnapshot[A, B] => + checkCompatibility(legacyConfig) case _ => CompatibilityResult.requiresMigration() } } + + private def checkCompatibility( + configSnapshot: CompositeTypeSerializerConfigSnapshot[_] + ): CompatibilityResult[T] = { + + val previousLeftRightSerWithConfigs = + configSnapshot.getNestedSerializersAndConfigs + + val leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( + previousLeftRightSerWithConfigs.get(0).f0, + classOf[UnloadableDummyTypeSerializer[_]], + previousLeftRightSerWithConfigs.get(0).f1, + leftSerializer) + + val rightCompatResult = CompatibilityUtil.resolveCompatibilityResult( + previousLeftRightSerWithConfigs.get(1).f0, + classOf[UnloadableDummyTypeSerializer[_]], + previousLeftRightSerWithConfigs.get(1).f1, + rightSerializer) + + if (leftCompatResult.isRequiresMigration + || rightCompatResult.isRequiresMigration) { + CompatibilityResult.requiresMigration() + } else { + CompatibilityResult.compatible() + } + } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala index abc56d133e074..b7e21ceaaee2a 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializer.scala @@ -84,7 +84,7 @@ class EnumValueSerializer[E <: Enumeration](val enum: E) extends TypeSerializer[ } override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[E#Value] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[E#Value] = { configSnapshot match { case enumSerializerConfigSnapshot: EnumValueSerializer.ScalaEnumSerializerConfigSnapshot[_] => @@ -122,7 +122,7 @@ class EnumValueSerializer[E <: Enumeration](val enum: E) extends TypeSerializer[ object EnumValueSerializer { class ScalaEnumSerializerConfigSnapshot[E <: Enumeration] - extends TypeSerializerConfigSnapshot { + extends TypeSerializerConfigSnapshot[E#Value] { var enumClass: Class[E] = _ var enumConstants: List[(String, Int)] = _ diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala index 01ca29594e2a2..eff57b68755b7 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/NothingSerializer.scala @@ -56,11 +56,11 @@ class NothingSerializer extends TypeSerializer[Any] { override def deserialize(reuse: Any, source: DataInputView): Any = throw new RuntimeException("This must not be used. You encountered a bug.") - override def snapshotConfiguration(): TypeSerializerConfigSnapshot = + override def snapshotConfiguration(): TypeSerializerConfigSnapshot[Any] = throw new RuntimeException("This must not be used. You encountered a bug.") override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[Any] = + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Any] = throw new RuntimeException("This must not be used. You encountered a bug.") override def equals(obj: Any): Boolean = { diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala index aa4a0ea75b8f9..a34826f2765ee 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala @@ -106,7 +106,7 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A]) } override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[Option[A]] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Option[A]] = { configSnapshot match { case optionSerializerConfigSnapshot @@ -120,7 +120,7 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A]) } private def ensureCompatibility( - compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot) + compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot[Option[A]]) : CompatibilityResult[Option[A]] = { val compatResult = CompatibilityUtil.resolveCompatibilityResult( @@ -150,7 +150,7 @@ object OptionSerializer { * Once Flink 1.3.x is no longer supported, this can be removed. */ class OptionSerializerConfigSnapshot[A]() - extends CompositeTypeSerializerConfigSnapshot { + extends CompositeTypeSerializerConfigSnapshot[Option[A]] { override def getVersion: Int = OptionSerializerConfigSnapshot.VERSION } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala index b54193b48291f..b5d069fbe9396 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala @@ -152,16 +152,16 @@ abstract class TraversableSerializer[T <: TraversableOnce[E], E]( obj.isInstanceOf[TraversableSerializer[_, _]] } - override def snapshotConfiguration(): TraversableSerializerConfigSnapshot[E] = { - new TraversableSerializerConfigSnapshot[E](elementSerializer) + override def snapshotConfiguration(): TraversableSerializerConfigSnapshot[T, E] = { + new TraversableSerializerConfigSnapshot[T, E](elementSerializer) } override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[T] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[T] = { configSnapshot match { case traversableSerializerConfigSnapshot - : TraversableSerializerConfigSnapshot[E] => + : TraversableSerializerConfigSnapshot[T, E] => val elemCompatRes = CompatibilityUtil.resolveCompatibilityResult( traversableSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f0, diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala index cc9c5ccede775..0e5fd97acf16e 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala @@ -109,7 +109,7 @@ class TrySerializer[A]( } override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[Try[A]] = { + configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Try[A]] = { configSnapshot match { case trySerializerConfigSnapshot @@ -123,7 +123,7 @@ class TrySerializer[A]( } private def ensureCompatibility( - compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot) + compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot[Try[A]]) : CompatibilityResult[Try[A]] = { val previousSerializersAndConfigs = @@ -156,7 +156,7 @@ object TrySerializer { * Once Flink 1.3.x is no longer supported, this can be removed. */ class TrySerializerConfigSnapshot[A]() - extends CompositeTypeSerializerConfigSnapshot() { + extends CompositeTypeSerializerConfigSnapshot[Try[A]]() { override def getVersion: Int = TrySerializerConfigSnapshot.VERSION } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java index c2ebdf483ed11..5b19cbe3915a8 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java @@ -553,15 +553,15 @@ public boolean canEqual(Object obj) { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot> snapshotConfiguration() { return new UnionSerializerConfigSnapshot<>(oneSerializer, twoSerializer); } @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof UnionSerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> previousSerializersAndConfigs = - ((UnionSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); + ((UnionSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult oneSerializerCompatResult = CompatibilityUtil.resolveCompatibilityResult( previousSerializersAndConfigs.get(0).f0, @@ -592,7 +592,8 @@ public CompatibilityResult> ensureCompatibility(TypeSerializ /** * The {@link TypeSerializerConfigSnapshot} for the {@link UnionSerializer}. */ - public static class UnionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { + public static class UnionSerializerConfigSnapshot + extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java index 2ffb6d5810ed5..275cfc7634447 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java @@ -763,16 +763,16 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot> snapshotConfiguration() { return new StateSerializerConfigSnapshot<>(transactionSerializer, contextSerializer); } @Override public CompatibilityResult> ensureCompatibility( - TypeSerializerConfigSnapshot configSnapshot) { + TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof StateSerializerConfigSnapshot) { List, TypeSerializerConfigSnapshot>> previousSerializersAndConfigs = - ((StateSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); + ((StateSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult txnCompatResult = CompatibilityUtil.resolveCompatibilityResult( previousSerializersAndConfigs.get(0).f0, @@ -809,7 +809,7 @@ public CompatibilityResult> ensureCompatibility( */ @Internal public static final class StateSerializerConfigSnapshot - extends CompositeTypeSerializerConfigSnapshot { + extends CompositeTypeSerializerConfigSnapshot> { private static final int VERSION = 1; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimerHeapInternalTimer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimerHeapInternalTimer.java index 906b0908abbf7..dd2709bfc4dc2 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimerHeapInternalTimer.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimerHeapInternalTimer.java @@ -242,12 +242,12 @@ public int hashCode() { } @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { + public TypeSerializerConfigSnapshot> snapshotConfiguration() { throw new UnsupportedOperationException("This serializer is not registered for managed state."); } @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { throw new UnsupportedOperationException("This serializer is not registered for managed state."); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java index ba92416d792ce..635851ed259c2 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java @@ -281,18 +281,18 @@ public int hashCode() { // -------------------------------------------------------------------------------------------- @Override - public StreamElementSerializerConfigSnapshot snapshotConfiguration() { + public StreamElementSerializerConfigSnapshot snapshotConfiguration() { return new StreamElementSerializerConfigSnapshot<>(typeSerializer); } @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { Tuple2, TypeSerializerConfigSnapshot> previousTypeSerializerAndConfig; // we are compatible for data written by ourselves or the legacy MultiplexingStreamRecordSerializer if (configSnapshot instanceof StreamElementSerializerConfigSnapshot) { previousTypeSerializerAndConfig = - ((StreamElementSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); + ((StreamElementSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); } else { return CompatibilityResult.requiresMigration(); } @@ -317,7 +317,7 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConf /** * Configuration snapshot specific to the {@link StreamElementSerializer}. */ - public static final class StreamElementSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { + public static final class StreamElementSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { private static final int VERSION = 1; From 661eb6d34da450ed096a77f166a4cc62ce3efdba Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 14 Jun 2018 11:52:06 +0200 Subject: [PATCH 2/6] [FLINK-9377] [core] (part 2) Remove fallback deserializer option from CompatibilityResult Now that the config snapshot is used as a factory for the restore serializer, it should be guaranteed that a restore serializer is always available. This removes the need for the user to provide a "fallback" convert serializer in the case where a migration is required. --- .../common/typeutils/CompatibilityResult.java | 68 ++----------------- .../common/typeutils/CompatibilityUtil.java | 21 +----- .../base/GenericArraySerializer.java | 9 --- .../common/typeutils/base/ListSerializer.java | 7 -- .../common/typeutils/base/MapSerializer.java | 11 --- .../typeutils/runtime/EitherSerializer.java | 13 ---- .../typeutils/runtime/PojoSerializer.java | 54 +++------------ .../java/typeutils/runtime/RowSerializer.java | 26 +------ .../runtime/TupleSerializerBase.java | 26 +------ .../java/org/apache/flink/cep/nfa/NFA.java | 24 ++----- .../apache/flink/cep/nfa/SharedBuffer.java | 31 ++------- .../table/dataview/ListViewSerializer.scala | 2 - .../table/dataview/MapViewSerializer.scala | 4 -- .../table/runtime/types/CRowSerializer.scala | 11 +-- .../runtime/state/ArrayListSerializer.java | 7 -- .../state/DefaultOperatorStateBackend.java | 6 -- .../RegisteredKeyedBackendStateMetaInfo.java | 5 -- .../state/heap/HeapKeyedStateBackend.java | 3 - .../scala/typeutils/EitherSerializer.scala | 4 -- .../scala/typeutils/OptionSerializer.scala | 10 +-- .../typeutils/TraversableSerializer.scala | 2 - .../api/scala/typeutils/TrySerializer.scala | 4 -- .../state/RocksDBKeyedStateBackend.java | 5 -- .../api/datastream/CoGroupedStreams.java | 11 --- .../sink/TwoPhaseCommitSinkFunction.java | 13 ---- .../operators/HeapInternalTimerService.java | 4 -- .../streamrecord/StreamElementSerializer.java | 8 --- 27 files changed, 36 insertions(+), 353 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java index 1e05d57854b96..992b54a7dbf76 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityResult.java @@ -19,9 +19,6 @@ package org.apache.flink.api.common.typeutils; import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.util.Preconditions; - -import javax.annotation.Nonnull; /** * A {@code CompatibilityResult} contains information about whether or not data migration @@ -35,79 +32,26 @@ public final class CompatibilityResult { /** Whether or not migration is required. */ private final boolean requiresMigration; - /** - * The convert deserializer to use for reading previous data during migration, - * in the case that the preceding serializer cannot be found. - * - *

This is only relevant if migration is required. - */ - private final TypeDeserializer convertDeserializer; - /** * Returns a result that signals that the new serializer is compatible and no migration is required. * - * @return a result that signals migration is not required for the new serializer + * @return a result that signals migration is not required for the new serializer. */ public static CompatibilityResult compatible() { - return new CompatibilityResult<>(false, null); - } - - /** - * Returns a result that signals migration to be performed, and in the case that the preceding serializer - * cannot be found or restored to read the previous data during migration, a provided convert deserializer - * can be used. - * - * @param convertDeserializer the convert deserializer to use, in the case that the preceding serializer - * cannot be found. - * - * @param the type of the data being migrated. - * - * @return a result that signals migration is necessary, also providing a convert deserializer. - */ - public static CompatibilityResult requiresMigration(@Nonnull TypeDeserializer convertDeserializer) { - Preconditions.checkNotNull(convertDeserializer, "Convert deserializer cannot be null."); - - return new CompatibilityResult<>(true, convertDeserializer); - } - - /** - * Returns a result that signals migration to be performed, and in the case that the preceding serializer - * cannot be found or restored to read the previous data during migration, a provided convert serializer - * can be used. The provided serializer will only be used for deserialization. - * - * @param convertSerializer the convert serializer to use, in the case that the preceding serializer - * cannot be found. The provided serializer will only be used for deserialization. - * - * @param the type of the data being migrated. - * - * @return a result that signals migration is necessary, also providing a convert serializer. - */ - public static CompatibilityResult requiresMigration(@Nonnull TypeSerializer convertSerializer) { - Preconditions.checkNotNull(convertSerializer, "Convert serializer cannot be null."); - - return new CompatibilityResult<>(true, new TypeDeserializerAdapter<>(convertSerializer)); + return new CompatibilityResult<>(false); } /** - * Returns a result that signals migration to be performed. The migration will fail if the preceding - * serializer for the previous data cannot be found. + * Returns a result that signals migration to be performed. * - *

You can also provide a convert deserializer using {@link #requiresMigration(TypeDeserializer)} - * or {@link #requiresMigration(TypeSerializer)}, which will be used as a fallback resort in such cases. - * - * @return a result that signals migration is necessary, without providing a convert deserializer. + * @return a result that signals migration is necessary. */ public static CompatibilityResult requiresMigration() { - return new CompatibilityResult<>(true, null); + return new CompatibilityResult<>(true); } - private CompatibilityResult(boolean requiresMigration, TypeDeserializer convertDeserializer) { + private CompatibilityResult(boolean requiresMigration) { this.requiresMigration = requiresMigration; - this.convertDeserializer = convertDeserializer; - } - - public TypeDeserializer getConvertDeserializer() { - return convertDeserializer; } public boolean isRequiresMigration() { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java index 6c8583c196815..cff464943b934 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java @@ -19,8 +19,6 @@ import org.apache.flink.annotation.Internal; -import javax.annotation.Nullable; - /** * Utilities related to serializer compatibility. */ @@ -44,8 +42,6 @@ public class CompatibilityUtil { * If yes, use that for state migration and simply return the result. * 6. If all of above fails, state migration is required but could not be performed; throw exception. * - * @param precedingSerializer the preceding serializer used to write the data, null if none could be retrieved - * @param dummySerializerClassTag any class tags that identifies the preceding serializer as a dummy placeholder * @param precedingSerializerConfigSnapshot configuration snapshot of the preceding serializer * @param newSerializer the new serializer to ensure compatibility with * @@ -55,26 +51,11 @@ public class CompatibilityUtil { */ @SuppressWarnings("unchecked") public static CompatibilityResult resolveCompatibilityResult( - @Nullable TypeSerializer precedingSerializer, - Class dummySerializerClassTag, TypeSerializerConfigSnapshot precedingSerializerConfigSnapshot, TypeSerializer newSerializer) { if (precedingSerializerConfigSnapshot != null) { - CompatibilityResult initialResult = newSerializer.ensureCompatibility(precedingSerializerConfigSnapshot); - - if (!initialResult.isRequiresMigration()) { - return initialResult; - } else { - if (precedingSerializer != null && !(precedingSerializer.getClass().equals(dummySerializerClassTag))) { - // if the preceding serializer exists and is not a dummy, use - // that for converting instead of any provided convert deserializer - return CompatibilityResult.requiresMigration((TypeSerializer) precedingSerializer); - } else { - // requires migration (may or may not have a convert deserializer) - return initialResult; - } - } + return newSerializer.ensureCompatibility(precedingSerializerConfigSnapshot); } else { // if the configuration snapshot of the preceding serializer cannot be provided, // we can only simply assume that the new serializer is compatible diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java index a42226cd91146..8e0969f350929 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java @@ -24,10 +24,8 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -212,18 +210,11 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot config.getSingleNestedSerializerAndConfig(); CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousComponentSerializerAndConfig.f0, - UnloadableDummyTypeSerializer.class, previousComponentSerializerAndConfig.f1, componentSerializer); if (!compatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else if (compatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new GenericArraySerializer<>( - componentClass, - new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()))); } } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java index b5759740c043a..1ae0250ce2802 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java @@ -21,10 +21,8 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -190,16 +188,11 @@ public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnap ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousElemSerializerAndConfig.f0, - UnloadableDummyTypeSerializer.class, previousElemSerializerAndConfig.f1, elementSerializer); if (!compatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else if (compatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new ListSerializer<>(new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()))); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java index 570f8f83a2f74..e010a905281ae 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java @@ -21,10 +21,8 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -218,24 +216,15 @@ public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSn ((MapSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(0).f0, - UnloadableDummyTypeSerializer.class, previousKvSerializersAndConfigs.get(0).f1, keySerializer); CompatibilityResult valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(1).f0, - UnloadableDummyTypeSerializer.class, previousKvSerializersAndConfigs.get(1).f1, valueSerializer); if (!keyCompatResult.isRequiresMigration() && !valueCompatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else if (keyCompatResult.getConvertDeserializer() != null && valueCompatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new MapSerializer<>( - new TypeDeserializerAdapter<>(keyCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(valueCompatResult.getConvertDeserializer()))); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java index 4f27589e41327..4456b4de83286 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java @@ -21,10 +21,8 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -207,26 +205,15 @@ public CompatibilityResult> ensureCompatibility(TypeSerializerConfi ((EitherSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerializersAndConfigs.get(0).f0, - UnloadableDummyTypeSerializer.class, previousLeftRightSerializersAndConfigs.get(0).f1, leftSerializer); CompatibilityResult rightCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerializersAndConfigs.get(1).f0, - UnloadableDummyTypeSerializer.class, previousLeftRightSerializersAndConfigs.get(1).f1, rightSerializer); if (!leftCompatResult.isRequiresMigration() && !rightCompatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else { - if (leftCompatResult.getConvertDeserializer() != null && rightCompatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new EitherSerializer<>( - new TypeDeserializerAdapter<>(leftCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(rightCompatResult.getConvertDeserializer()))); - } } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java index cdff1fa0aeca0..39b91245b419d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java @@ -39,7 +39,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; @@ -598,8 +597,6 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot config = (PojoSerializerConfigSnapshot) configSnapshot; - boolean requiresMigration = false; - if (clazz.equals(config.getTypeClass())) { if (this.numFields == config.getFieldToSerializerConfigSnapshot().size()) { @@ -622,19 +619,11 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot) compatResult.getConvertDeserializer(); - } else { - return CompatibilityResult.requiresMigration(); - } + return CompatibilityResult.requiresMigration(); } else { reorderedFieldSerializers[i] = fieldSerializers[fieldIndex]; } @@ -671,17 +660,11 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot, TypeSerializerConfigSnapshot> previousRegisteredSerializerConfig : previousRegistrations.values()) { // check compatibility of subclass serializer compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousRegisteredSerializerConfig.f0, - UnloadableDummyTypeSerializer.class, previousRegisteredSerializerConfig.f1, reorderedRegisteredSubclassSerializers[i]); if (compatResult.isRequiresMigration()) { - requiresMigration = true; - - if (compatResult.getConvertDeserializer() == null) { - return CompatibilityResult.requiresMigration(); - } + return CompatibilityResult.requiresMigration(); } i++; @@ -700,19 +683,11 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot ensureCompatibility(TypeSerializerConfigSnapshot( - clazz, - reorderedFields, - reorderedFieldSerializers, - reorderedRegisteredSubclassesToClasstags, - reorderedRegisteredSubclassSerializers, - rebuiltCache)); - } + return CompatibilityResult.compatible(); } } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java index b806c27a97ea9..a7c3b18a4ae4f 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java @@ -21,10 +21,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -274,38 +272,20 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot ((RowSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); if (previousFieldSerializersAndConfigs.size() == fieldSerializers.length) { - boolean requireMigration = false; - TypeSerializer[] convertDeserializers = new TypeSerializer[fieldSerializers.length]; CompatibilityResult compatResult; int i = 0; for (Tuple2, TypeSerializerConfigSnapshot> f : previousFieldSerializersAndConfigs) { - compatResult = CompatibilityUtil.resolveCompatibilityResult( - f.f0, - UnloadableDummyTypeSerializer.class, - f.f1, - fieldSerializers[i]); + compatResult = CompatibilityUtil.resolveCompatibilityResult(f.f1, fieldSerializers[i]); if (compatResult.isRequiresMigration()) { - requireMigration = true; - - if (compatResult.getConvertDeserializer() == null) { - // one of the field serializers cannot provide a fallback deserializer - return CompatibilityResult.requiresMigration(); - } else { - convertDeserializers[i] = - new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()); - } + return CompatibilityResult.requiresMigration(); } i++; } - if (requireMigration) { - return CompatibilityResult.requiresMigration(new RowSerializer(convertDeserializers)); - } else { - return CompatibilityResult.compatible(); - } + return CompatibilityResult.compatible(); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java index 34146ca8e022a..fdcd828ca3643 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java @@ -22,10 +22,8 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -145,37 +143,19 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot[] convertFieldSerializers = new TypeSerializer[fieldSerializers.length]; - boolean requiresMigration = false; CompatibilityResult compatResult; int i = 0; for (Tuple2, TypeSerializerConfigSnapshot> f : previousFieldSerializersAndConfigs) { - compatResult = CompatibilityUtil.resolveCompatibilityResult( - f.f0, - UnloadableDummyTypeSerializer.class, - f.f1, - fieldSerializers[i]); + compatResult = CompatibilityUtil.resolveCompatibilityResult(f.f1, fieldSerializers[i]); if (compatResult.isRequiresMigration()) { - requiresMigration = true; - - if (compatResult.getConvertDeserializer() != null) { - convertFieldSerializers[i] = - new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()); - } else { - return CompatibilityResult.requiresMigration(); - } + return CompatibilityResult.requiresMigration(); } i++; } - if (!requiresMigration) { - return CompatibilityResult.compatible(); - } else { - return CompatibilityResult.requiresMigration( - createSerializerInstance(tupleClass, convertFieldSerializers)); - } + return CompatibilityResult.compatible(); } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java index a6ae7746812fa..1fe4a865fae7c 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java @@ -22,10 +22,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.cep.nfa.compiler.NFACompiler; @@ -998,28 +996,16 @@ public CompatibilityResult> ensureCompatibility(TypeSerializerCon ((NFASerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult eventCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializersAndConfigs.get(0).f0, - UnloadableDummyTypeSerializer.class, - serializersAndConfigs.get(0).f1, - eventSerializer); + serializersAndConfigs.get(0).f1, + eventSerializer); CompatibilityResult> sharedBufCompatResult = - CompatibilityUtil.resolveCompatibilityResult( - serializersAndConfigs.get(1).f0, - UnloadableDummyTypeSerializer.class, - serializersAndConfigs.get(1).f1, - sharedBufferSerializer); + CompatibilityUtil.resolveCompatibilityResult( + serializersAndConfigs.get(1).f1, + sharedBufferSerializer); if (!sharedBufCompatResult.isRequiresMigration() && !eventCompatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else { - if (eventCompatResult.getConvertDeserializer() != null && - sharedBufCompatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new NFASerializer<>( - new TypeDeserializerAdapter<>(eventCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(sharedBufCompatResult.getConvertDeserializer()))); - } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java index 4ab227961b0a4..6fc50bc86bd32 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java @@ -21,10 +21,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.cep.nfa.compiler.NFAStateNameHandler; import org.apache.flink.cep.nfa.sharedbuffer.EventId; @@ -357,37 +355,22 @@ public CompatibilityResult> ensureCompatibility(TypeSerializerCo ((SharedBufferSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializerConfigSnapshots.get(0).f0, - UnloadableDummyTypeSerializer.class, - serializerConfigSnapshots.get(0).f1, - keySerializer); + serializerConfigSnapshots.get(0).f1, + keySerializer); CompatibilityResult valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializerConfigSnapshots.get(1).f0, - UnloadableDummyTypeSerializer.class, - serializerConfigSnapshots.get(1).f1, - valueSerializer); + serializerConfigSnapshots.get(1).f1, + valueSerializer); CompatibilityResult versionCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializerConfigSnapshots.get(2).f0, - UnloadableDummyTypeSerializer.class, - serializerConfigSnapshots.get(2).f1, - versionSerializer); + serializerConfigSnapshots.get(2).f1, + versionSerializer); if (!keyCompatResult.isRequiresMigration() && !valueCompatResult.isRequiresMigration() && !versionCompatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); } else { - if (keyCompatResult.getConvertDeserializer() != null - && valueCompatResult.getConvertDeserializer() != null - && versionCompatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new SharedBufferSerializer<>( - new TypeDeserializerAdapter<>(keyCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(valueCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(versionCompatResult.getConvertDeserializer()) - )); - } + return CompatibilityResult.requiresMigration(); } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala index 71ef183943f27..b8f631fd067f1 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala @@ -102,8 +102,6 @@ class ListViewSerializer[T](val listSerializer: ListSerializer[T]) val previousListSerializerAndConfig = configSnapshot.getSingleNestedSerializerAndConfig val compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousListSerializerAndConfig.f0, - classOf[UnloadableDummyTypeSerializer[_]], previousListSerializerAndConfig.f1, listSerializer.getElementSerializer) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala index 0a98e34ede0d2..d624a40944c85 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala @@ -105,14 +105,10 @@ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) val previousKvSerializersAndConfigs = configSnapshot.getNestedSerializersAndConfigs val keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(0).f0, - classOf[UnloadableDummyTypeSerializer[_]], previousKvSerializersAndConfigs.get(0).f1, mapSerializer.getKeySerializer) val valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(1).f0, - classOf[UnloadableDummyTypeSerializer[_]], previousKvSerializersAndConfigs.get(1).f1, mapSerializer.getValueSerializer) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala index e96da82dc3bb9..a7f1f4748bb8d 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala @@ -90,20 +90,11 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali configSnapshot match { case crowSerializerConfigSnapshot: CRowSerializer.CRowSerializerConfigSnapshot => val compatResult = CompatibilityUtil.resolveCompatibilityResult( - crowSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f0, - classOf[UnloadableDummyTypeSerializer[_]], crowSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f1, rowSerializer) if (compatResult.isRequiresMigration) { - if (compatResult.getConvertDeserializer != null) { - CompatibilityResult.requiresMigration( - new CRowSerializer( - new TypeDeserializerAdapter(compatResult.getConvertDeserializer)) - ) - } else { - CompatibilityResult.requiresMigration() - } + CompatibilityResult.requiresMigration() } else { CompatibilityResult.compatible() } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java index 1868abf311c16..d30864c6c5d89 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java @@ -19,10 +19,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.CollectionSerializerConfigSnapshot; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; @@ -156,16 +154,11 @@ public CompatibilityResult> ensureCompatibility(TypeSerializerConfi ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousElemSerializerAndConfig.f0, - UnloadableDummyTypeSerializer.class, previousElemSerializerAndConfig.f1, elementSerializer); if (!compatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else if (compatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new ArrayListSerializer<>(new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()))); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java index a2e49cb179d7f..58083b8c87612 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java @@ -226,14 +226,10 @@ public BroadcastState getBroadcastState(final MapStateDescriptor keyCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredMetaInfo.getKeySerializer(), - UnloadableDummyTypeSerializer.class, restoredMetaInfo.getKeySerializerConfigSnapshot(), broadcastStateKeySerializer); CompatibilityResult valueCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredMetaInfo.getValueSerializer(), - UnloadableDummyTypeSerializer.class, restoredMetaInfo.getValueSerializerConfigSnapshot(), broadcastStateValueSerializer); @@ -759,8 +755,6 @@ private ListState getListState( // check compatibility to determine if state migration is required TypeSerializer newPartitionStateSerializer = partitionStateSerializer.duplicate(); CompatibilityResult stateCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredMetaInfo.getPartitionStateSerializer(), - UnloadableDummyTypeSerializer.class, restoredMetaInfo.getPartitionStateSerializerConfigSnapshot(), newPartitionStateSerializer); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java index c7952ef985ca6..3f7a8d6f8247e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java @@ -23,7 +23,6 @@ import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.util.Preconditions; import org.apache.flink.util.StateMigrationException; @@ -273,15 +272,11 @@ public static RegisteredKeyedBackendStateMetaInfo resolveKvStateCo // check compatibility results to determine if state migration is required CompatibilityResult namespaceCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredStateMetaInfoSnapshot.getNamespaceSerializer(), - null, restoredStateMetaInfoSnapshot.getNamespaceSerializerConfigSnapshot(), newNamespaceSerializer); TypeSerializer newStateSerializer = newStateDescriptor.getSerializer(); CompatibilityResult stateCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredStateMetaInfoSnapshot.getStateSerializer(), - UnloadableDummyTypeSerializer.class, restoredStateMetaInfoSnapshot.getStateSerializerConfigSnapshot(), newStateSerializer); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index 82ce5847627a1..87c3b57df541c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -30,7 +30,6 @@ import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.fs.FSDataInputStream; import org.apache.flink.core.memory.DataInputViewStreamWrapper; @@ -292,8 +291,6 @@ private void restorePartitionedState(Collection state) throws // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( - serializationProxy.getKeySerializer(), - UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), keySerializer) .isRequiresMigration()) { diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala index 82637befdcd1a..5049da45eb92e 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala @@ -140,14 +140,10 @@ class EitherSerializer[A, B, T <: Either[A, B]]( configSnapshot.getNestedSerializersAndConfigs val leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerWithConfigs.get(0).f0, - classOf[UnloadableDummyTypeSerializer[_]], previousLeftRightSerWithConfigs.get(0).f1, leftSerializer) val rightCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerWithConfigs.get(1).f0, - classOf[UnloadableDummyTypeSerializer[_]], previousLeftRightSerWithConfigs.get(1).f1, rightSerializer) diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala index a34826f2765ee..c7ac4e95081d6 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala @@ -124,19 +124,11 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A]) : CompatibilityResult[Option[A]] = { val compatResult = CompatibilityUtil.resolveCompatibilityResult( - compositeConfigSnapshot.getSingleNestedSerializerAndConfig.f0, - classOf[UnloadableDummyTypeSerializer[_]], compositeConfigSnapshot.getSingleNestedSerializerAndConfig.f1, elemSerializer) if (compatResult.isRequiresMigration) { - if (compatResult.getConvertDeserializer != null) { - CompatibilityResult.requiresMigration( - new OptionSerializer[A]( - new TypeDeserializerAdapter(compatResult.getConvertDeserializer))) - } else { - CompatibilityResult.requiresMigration() - } + CompatibilityResult.requiresMigration() } else { CompatibilityResult.compatible() } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala index b5d069fbe9396..574dc0474c4aa 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala @@ -164,8 +164,6 @@ abstract class TraversableSerializer[T <: TraversableOnce[E], E]( : TraversableSerializerConfigSnapshot[T, E] => val elemCompatRes = CompatibilityUtil.resolveCompatibilityResult( - traversableSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f0, - classOf[UnloadableDummyTypeSerializer[_]], traversableSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f1, elementSerializer) diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala index 0e5fd97acf16e..8f4c2a2c3447d 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala @@ -130,14 +130,10 @@ class TrySerializer[A]( compositeConfigSnapshot.getNestedSerializersAndConfigs val elemCompatRes = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(0).f0, - classOf[UnloadableDummyTypeSerializer[_]], previousSerializersAndConfigs.get(0).f1, elemSerializer) val throwableCompatRes = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(1).f0, - classOf[UnloadableDummyTypeSerializer[_]], previousSerializersAndConfigs.get(1).f1, throwableSerializer) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index e5f443a95ce8b..a7b8b4cb4580d 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -29,7 +29,6 @@ import org.apache.flink.api.common.state.ValueStateDescriptor; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ConfigConstants; @@ -612,8 +611,6 @@ private void restoreKVStateMetaData() throws IOException, StateMigrationExceptio // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( - serializationProxy.getKeySerializer(), - UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), rocksDBKeyedStateBackend.keySerializer) .isRequiresMigration()) { @@ -1156,8 +1153,6 @@ private void restoreInstanceDirectoryFromPath(Path source) throws IOException { // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( - serializationProxy.getKeySerializer(), - UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), stateBackend.keySerializer) .isRequiresMigration()) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java index 5b19cbe3915a8..01122c9e215f0 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java @@ -27,10 +27,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.operators.translation.WrappingFunction; import org.apache.flink.api.java.tuple.Tuple2; @@ -564,24 +562,15 @@ public CompatibilityResult> ensureCompatibility(TypeSerializ ((UnionSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult oneSerializerCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(0).f0, - UnloadableDummyTypeSerializer.class, previousSerializersAndConfigs.get(0).f1, oneSerializer); CompatibilityResult twoSerializerCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(1).f0, - UnloadableDummyTypeSerializer.class, previousSerializersAndConfigs.get(1).f1, twoSerializer); if (!oneSerializerCompatResult.isRequiresMigration() && !twoSerializerCompatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else if (oneSerializerCompatResult.getConvertDeserializer() != null && twoSerializerCompatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new UnionSerializer<>( - new TypeDeserializerAdapter<>(oneSerializerCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(twoSerializerCompatResult.getConvertDeserializer()))); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java index 275cfc7634447..3e41ae23aa621 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java @@ -27,10 +27,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -775,26 +773,15 @@ public CompatibilityResult> ensureCompatibility( ((StateSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); CompatibilityResult txnCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(0).f0, - UnloadableDummyTypeSerializer.class, previousSerializersAndConfigs.get(0).f1, transactionSerializer); CompatibilityResult contextCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(1).f0, - UnloadableDummyTypeSerializer.class, previousSerializersAndConfigs.get(1).f1, contextSerializer); if (!txnCompatResult.isRequiresMigration() && !contextCompatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else { - if (txnCompatResult.getConvertDeserializer() != null && contextCompatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new StateSerializer<>( - new TypeDeserializerAdapter<>(txnCompatResult.getConvertDeserializer()), - new TypeDeserializerAdapter<>(contextCompatResult.getConvertDeserializer()))); - } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java index c5a68fb912a4a..be85748bd8625 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java @@ -137,14 +137,10 @@ public void startTimerService( // the following is the case where we restore if (restoredTimersSnapshot != null) { CompatibilityResult keySerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult( - this.keyDeserializer, - null, restoredTimersSnapshot.getKeySerializerConfigSnapshot(), keySerializer); CompatibilityResult namespaceSerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult( - this.namespaceDeserializer, - null, restoredTimersSnapshot.getNamespaceSerializerConfigSnapshot(), namespaceSerializer); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java index 635851ed259c2..93366d47d41b2 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java @@ -22,10 +22,8 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.CompatibilityUtil; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeDeserializerAdapter; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -298,17 +296,11 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConf } CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousTypeSerializerAndConfig.f0, - UnloadableDummyTypeSerializer.class, previousTypeSerializerAndConfig.f1, typeSerializer); if (!compatResult.isRequiresMigration()) { return CompatibilityResult.compatible(); - } else if (compatResult.getConvertDeserializer() != null) { - return CompatibilityResult.requiresMigration( - new StreamElementSerializer<>( - new TypeDeserializerAdapter<>(compatResult.getConvertDeserializer()))); } else { return CompatibilityResult.requiresMigration(); } From c91d045c5eb6e355981e4edaa6d1a0d48e5d4a5e Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 14 Jun 2018 16:41:45 +0200 Subject: [PATCH 3/6] [FLINK-9377] [core] (part 3) Deprecate TypeSerializerSerializationUtil This commit deprecates all utility methods and classes related to serializing serializers. All methods that will still be in use, i.e. writing config snapshots, are now moved to a separate new TypeSerializerConfigSnapshotSerializationUtil class. --- ...alizerConfigSnapshotSerializationUtil.java | 185 ++++++++++++++++++ .../TypeSerializerSerializationUtil.java | 160 +-------------- .../typeutils/runtime/PojoSerializer.java | 13 +- .../common/typeutils/SerializerTestBase.java | 4 +- .../TypeSerializerSerializationUtilTest.java | 19 +- .../typeutils/base/EnumSerializerTest.java | 6 +- .../base/EnumSerializerUpgradeTest.java | 6 +- .../typeutils/runtime/PojoSerializerTest.java | 22 +-- .../kryo/KryoSerializerCompatibilityTest.java | 11 +- .../EnumValueSerializerUpgradeTest.scala | 6 +- 10 files changed, 237 insertions(+), 195 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotSerializationUtil.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotSerializationUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotSerializationUtil.java new file mode 100644 index 0000000000000..d7d4e16e45c8f --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerConfigSnapshotSerializationUtil.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils; + +import org.apache.flink.core.io.VersionedIOReadableWritable; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.InstantiationUtil; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Utility methods for serialization of {@link TypeSerializerConfigSnapshot}. + */ +public class TypeSerializerConfigSnapshotSerializationUtil { + + /** + * Writes a {@link TypeSerializerConfigSnapshot} to the provided data output view. + * + *

It is written with a format that can be later read again using + * {@link #readSerializerConfigSnapshot(DataInputView, ClassLoader)}. + * + * @param out the data output view + * @param serializerConfigSnapshot the serializer configuration snapshot to write + * + * @throws IOException + */ + public static void writeSerializerConfigSnapshot( + DataOutputView out, + TypeSerializerConfigSnapshot serializerConfigSnapshot) throws IOException { + + new TypeSerializerConfigSnapshotSerializationProxy(serializerConfigSnapshot).write(out); + } + + /** + * Reads from a data input view a {@link TypeSerializerConfigSnapshot} that was previously + * written using {@link TypeSerializerConfigSnapshotSerializationUtil#writeSerializerConfigSnapshot(DataOutputView, TypeSerializerConfigSnapshot)}. + * + * @param in the data input view + * @param userCodeClassLoader the user code class loader to use + * + * @return the read serializer configuration snapshot + * + * @throws IOException + */ + public static TypeSerializerConfigSnapshot readSerializerConfigSnapshot( + DataInputView in, + ClassLoader userCodeClassLoader) throws IOException { + + final TypeSerializerConfigSnapshotSerializationProxy proxy = new TypeSerializerConfigSnapshotSerializationProxy(userCodeClassLoader); + proxy.read(in); + + return proxy.getSerializerConfigSnapshot(); + } + + /** + * Writes multiple {@link TypeSerializerConfigSnapshot}s to the provided data output view. + * + *

It is written with a format that can be later read again using + * {@link #readSerializerConfigSnapshots(DataInputView, ClassLoader)}. + * + * @param out the data output view + * @param serializerConfigSnapshots the serializer configuration snapshots to write + * + * @throws IOException + */ + public static void writeSerializerConfigSnapshots( + DataOutputView out, + List> serializerConfigSnapshots) throws IOException { + + out.writeInt(serializerConfigSnapshots.size()); + + for (TypeSerializerConfigSnapshot snapshot : serializerConfigSnapshots) { + new TypeSerializerConfigSnapshotSerializationProxy(snapshot).write(out); + } + } + + /** + * Reads from a data input view multiple {@link TypeSerializerConfigSnapshot}s that was previously + * written using {@link TypeSerializerConfigSnapshotSerializationUtil#writeSerializerConfigSnapshot(DataOutputView, TypeSerializerConfigSnapshot)}. + * + * @param in the data input view + * @param userCodeClassLoader the user code class loader to use + * + * @return the read serializer configuration snapshots + * + * @throws IOException + */ + public static List> readSerializerConfigSnapshots( + DataInputView in, + ClassLoader userCodeClassLoader) throws IOException { + + int numFields = in.readInt(); + final List> serializerConfigSnapshots = new ArrayList<>(numFields); + + TypeSerializerConfigSnapshotSerializationProxy proxy; + for (int i = 0; i < numFields; i++) { + proxy = new TypeSerializerConfigSnapshotSerializationProxy(userCodeClassLoader); + proxy.read(in); + serializerConfigSnapshots.add(proxy.getSerializerConfigSnapshot()); + } + + return serializerConfigSnapshots; + } + + /** + * Utility serialization proxy for a {@link TypeSerializerConfigSnapshot}. + */ + static final class TypeSerializerConfigSnapshotSerializationProxy extends VersionedIOReadableWritable { + + private static final int VERSION = 1; + + private ClassLoader userCodeClassLoader; + private TypeSerializerConfigSnapshot serializerConfigSnapshot; + + TypeSerializerConfigSnapshotSerializationProxy(ClassLoader userCodeClassLoader) { + this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader); + } + + TypeSerializerConfigSnapshotSerializationProxy(TypeSerializerConfigSnapshot serializerConfigSnapshot) { + this.serializerConfigSnapshot = serializerConfigSnapshot; + } + + @Override + public void write(DataOutputView out) throws IOException { + super.write(out); + + // config snapshot class, so that we can re-instantiate the + // correct type of config snapshot instance when deserializing + out.writeUTF(serializerConfigSnapshot.getClass().getName()); + + // the actual configuration parameters + serializerConfigSnapshot.write(out); + } + + @SuppressWarnings("unchecked") + @Override + public void read(DataInputView in) throws IOException { + super.read(in); + + String serializerConfigClassname = in.readUTF(); + Class serializerConfigSnapshotClass; + try { + serializerConfigSnapshotClass = (Class) + Class.forName(serializerConfigClassname, true, userCodeClassLoader); + } catch (ClassNotFoundException e) { + throw new IOException( + "Could not find requested TypeSerializerConfigSnapshot class " + + serializerConfigClassname + " in classpath.", e); + } + + serializerConfigSnapshot = InstantiationUtil.instantiate(serializerConfigSnapshotClass); + serializerConfigSnapshot.setUserCodeClassLoader(userCodeClassLoader); + serializerConfigSnapshot.read(in); + } + + @Override + public int getVersion() { + return VERSION; + } + + TypeSerializerConfigSnapshot getSerializerConfigSnapshot() { + return serializerConfigSnapshot; + } + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java index e83b8c71ec35b..f6258836b33fb 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java @@ -40,9 +40,14 @@ import java.util.List; /** - * Utility methods for serialization of {@link TypeSerializer} and {@link TypeSerializerConfigSnapshot}. + * Utility methods for serialization of {@link TypeSerializer}. + * + * @deprecated This utility class was used to write serializers into checkpoints. + * Starting from Flink 1.6.x, this should no longer happen, and therefore + * this class is deprecated. It remains here for backwards compatibility paths. */ @Internal +@Deprecated public class TypeSerializerSerializationUtil { private static final Logger LOG = LoggerFactory.getLogger(TypeSerializerSerializationUtil.class); @@ -154,7 +159,7 @@ public static void writeSerializersAndConfigsWithResilience( writeSerializer(bufferWrapper, serAndConfSnapshot.f0); out.writeInt(bufferWithPos.getPosition()); - writeSerializerConfigSnapshot(bufferWrapper, serAndConfSnapshot.f1); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(bufferWrapper, serAndConfSnapshot.f1); } out.writeInt(bufferWithPos.getPosition()); @@ -208,7 +213,7 @@ public static List, TypeSerializerConfigSnapshot>> read serializer = tryReadSerializer(bufferWrapper, userCodeClassLoader, true); bufferWithPos.setPosition(offsets[i * 2 + 1]); - configSnapshot = readSerializerConfigSnapshot(bufferWrapper, userCodeClassLoader); + configSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(bufferWrapper, userCodeClassLoader); serializersAndConfigSnapshots.add( new Tuple2, TypeSerializerConfigSnapshot>(serializer, configSnapshot)); @@ -218,95 +223,6 @@ public static List, TypeSerializerConfigSnapshot>> read return serializersAndConfigSnapshots; } - /** - * Writes a {@link TypeSerializerConfigSnapshot} to the provided data output view. - * - *

It is written with a format that can be later read again using - * {@link #readSerializerConfigSnapshot(DataInputView, ClassLoader)}. - * - * @param out the data output view - * @param serializerConfigSnapshot the serializer configuration snapshot to write - * - * @throws IOException - */ - public static void writeSerializerConfigSnapshot( - DataOutputView out, - TypeSerializerConfigSnapshot serializerConfigSnapshot) throws IOException { - - new TypeSerializerConfigSnapshotSerializationProxy(serializerConfigSnapshot).write(out); - } - - /** - * Reads from a data input view a {@link TypeSerializerConfigSnapshot} that was previously - * written using {@link #writeSerializerConfigSnapshot(DataOutputView, TypeSerializerConfigSnapshot)}. - * - * @param in the data input view - * @param userCodeClassLoader the user code class loader to use - * - * @return the read serializer configuration snapshot - * - * @throws IOException - */ - public static TypeSerializerConfigSnapshot readSerializerConfigSnapshot( - DataInputView in, - ClassLoader userCodeClassLoader) throws IOException { - - final TypeSerializerConfigSnapshotSerializationProxy proxy = new TypeSerializerConfigSnapshotSerializationProxy(userCodeClassLoader); - proxy.read(in); - - return proxy.getSerializerConfigSnapshot(); - } - - /** - * Writes multiple {@link TypeSerializerConfigSnapshot}s to the provided data output view. - * - *

It is written with a format that can be later read again using - * {@link #readSerializerConfigSnapshots(DataInputView, ClassLoader)}. - * - * @param out the data output view - * @param serializerConfigSnapshots the serializer configuration snapshots to write - * - * @throws IOException - */ - public static void writeSerializerConfigSnapshots( - DataOutputView out, - TypeSerializerConfigSnapshot... serializerConfigSnapshots) throws IOException { - - out.writeInt(serializerConfigSnapshots.length); - - for (TypeSerializerConfigSnapshot snapshot : serializerConfigSnapshots) { - new TypeSerializerConfigSnapshotSerializationProxy(snapshot).write(out); - } - } - - /** - * Reads from a data input view multiple {@link TypeSerializerConfigSnapshot}s that was previously - * written using {@link #writeSerializerConfigSnapshot(DataOutputView, TypeSerializerConfigSnapshot)}. - * - * @param in the data input view - * @param userCodeClassLoader the user code class loader to use - * - * @return the read serializer configuration snapshots - * - * @throws IOException - */ - public static TypeSerializerConfigSnapshot[] readSerializerConfigSnapshots( - DataInputView in, - ClassLoader userCodeClassLoader) throws IOException { - - int numFields = in.readInt(); - final TypeSerializerConfigSnapshot[] serializerConfigSnapshots = new TypeSerializerConfigSnapshot[numFields]; - - TypeSerializerConfigSnapshotSerializationProxy proxy; - for (int i = 0; i < numFields; i++) { - proxy = new TypeSerializerConfigSnapshotSerializationProxy(userCodeClassLoader); - proxy.read(in); - serializerConfigSnapshots[i] = proxy.getSerializerConfigSnapshot(); - } - - return serializerConfigSnapshots; - } - // ----------------------------------------------------------------------------------------------------- /** @@ -384,64 +300,4 @@ public int getVersion() { } } - /** - * Utility serialization proxy for a {@link TypeSerializerConfigSnapshot}. - */ - static final class TypeSerializerConfigSnapshotSerializationProxy extends VersionedIOReadableWritable { - - private static final int VERSION = 1; - - private ClassLoader userCodeClassLoader; - private TypeSerializerConfigSnapshot serializerConfigSnapshot; - - TypeSerializerConfigSnapshotSerializationProxy(ClassLoader userCodeClassLoader) { - this.userCodeClassLoader = Preconditions.checkNotNull(userCodeClassLoader); - } - - TypeSerializerConfigSnapshotSerializationProxy(TypeSerializerConfigSnapshot serializerConfigSnapshot) { - this.serializerConfigSnapshot = serializerConfigSnapshot; - } - - @Override - public void write(DataOutputView out) throws IOException { - super.write(out); - - // config snapshot class, so that we can re-instantiate the - // correct type of config snapshot instance when deserializing - out.writeUTF(serializerConfigSnapshot.getClass().getName()); - - // the actual configuration parameters - serializerConfigSnapshot.write(out); - } - - @SuppressWarnings("unchecked") - @Override - public void read(DataInputView in) throws IOException { - super.read(in); - - String serializerConfigClassname = in.readUTF(); - Class serializerConfigSnapshotClass; - try { - serializerConfigSnapshotClass = (Class) - Class.forName(serializerConfigClassname, true, userCodeClassLoader); - } catch (ClassNotFoundException e) { - throw new IOException( - "Could not find requested TypeSerializerConfigSnapshot class " - + serializerConfigClassname + " in classpath.", e); - } - - serializerConfigSnapshot = InstantiationUtil.instantiate(serializerConfigSnapshotClass); - serializerConfigSnapshot.setUserCodeClassLoader(userCodeClassLoader); - serializerConfigSnapshot.read(in); - } - - @Override - public int getVersion() { - return VERSION; - } - - TypeSerializerConfigSnapshot getSerializerConfigSnapshot() { - return serializerConfigSnapshot; - } - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java index 39b91245b419d..f507f8a3a7a89 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java @@ -38,6 +38,7 @@ import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; @@ -802,7 +803,7 @@ public void write(DataOutputView out) throws IOException { } out.writeInt(outWithPos.getPosition()); - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(outViewWrapper, entry.getValue().f1); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(outViewWrapper, entry.getValue().f1); } // --- write registered subclasses and their serializers, in registration order @@ -819,7 +820,7 @@ public void write(DataOutputView out) throws IOException { } out.writeInt(outWithPos.getPosition()); - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(outViewWrapper, entry.getValue().f1); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(outViewWrapper, entry.getValue().f1); } // --- write snapshot of non-registered subclass serializer cache @@ -836,7 +837,7 @@ public void write(DataOutputView out) throws IOException { } out.writeInt(outWithPos.getPosition()); - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(outViewWrapper, entry.getValue().f1); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(outViewWrapper, entry.getValue().f1); } out.writeInt(outWithPos.getPosition()); @@ -891,7 +892,7 @@ public void read(DataInputView in) throws IOException { fieldSerializer = TypeSerializerSerializationUtil.tryReadSerializer(inViewWrapper, getUserCodeClassLoader(), true); inWithPos.setPosition(fieldSerializerOffsets[i * 2 + 1]); - fieldSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot(inViewWrapper, getUserCodeClassLoader()); + fieldSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(inViewWrapper, getUserCodeClassLoader()); fieldToSerializerConfigSnapshot.put( fieldName, @@ -917,7 +918,7 @@ public void read(DataInputView in) throws IOException { registeredSubclassSerializer = TypeSerializerSerializationUtil.tryReadSerializer(inViewWrapper, getUserCodeClassLoader(), true); inWithPos.setPosition(registeredSerializerOffsets[i * 2 + 1]); - registeredSubclassSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot(inViewWrapper, getUserCodeClassLoader()); + registeredSubclassSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(inViewWrapper, getUserCodeClassLoader()); this.registeredSubclassesToSerializerConfigSnapshots.put( registeredSubclass, @@ -943,7 +944,7 @@ public void read(DataInputView in) throws IOException { cachedSubclassSerializer = TypeSerializerSerializationUtil.tryReadSerializer(inViewWrapper, getUserCodeClassLoader(), true); inWithPos.setPosition(cachedSerializerOffsets[i * 2 + 1]); - cachedSubclassSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot(inViewWrapper, getUserCodeClassLoader()); + cachedSubclassSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(inViewWrapper, getUserCodeClassLoader()); this.nonRegisteredSubclassesToSerializerConfigSnapshots.put( cachedSubclass, diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java index d32cb0cd34174..a2844d9cfe603 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java @@ -109,14 +109,14 @@ public void testSnapshotConfigurationAndReconfigure() throws Exception { byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot); serializedConfig = out.toByteArray(); } TypeSerializerConfigSnapshot restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - restoredConfig = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + restoredConfig = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java index a7136fcd18bd2..7f1e0f61574ad 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java @@ -156,23 +156,22 @@ public void testSerializeConfigurationSnapshots() throws Exception { byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshots( + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshots( new DataOutputViewStreamWrapper(out), - configSnapshot1, - configSnapshot2); + Arrays.asList(configSnapshot1, configSnapshot2)); serializedConfig = out.toByteArray(); } - TypeSerializerConfigSnapshot[] restoredConfigs; + List> restoredConfigs; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - restoredConfigs = TypeSerializerSerializationUtil.readSerializerConfigSnapshots( + restoredConfigs = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshots( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - assertEquals(2, restoredConfigs.length); - assertEquals(configSnapshot1, restoredConfigs[0]); - assertEquals(configSnapshot2, restoredConfigs[1]); + assertEquals(2, restoredConfigs.size()); + assertEquals(configSnapshot1, restoredConfigs.get(0)); + assertEquals(configSnapshot2, restoredConfigs.get(1)); } /** @@ -182,14 +181,14 @@ public void testSerializeConfigurationSnapshots() throws Exception { public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception { byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( new DataOutputViewStreamWrapper(out), new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar")); serializedConfig = out.toByteArray(); } try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { // read using a dummy classloader - TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null)); fail("Expected a ClassNotFoundException wrapped in IOException"); } catch (IOException expected) { diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java index e3ce3ee6fca35..de564869d36a5 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java @@ -21,7 +21,7 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.SerializerTestInstance; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.util.InstantiationUtil; @@ -95,14 +95,14 @@ public void testConfigurationSnapshotSerialization() throws Exception { byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( new DataOutputViewStreamWrapper(out), serializer.snapshotConfiguration()); serializedConfig = out.toByteArray(); } TypeSerializerConfigSnapshot restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - restoredConfig = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + restoredConfig = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java index 1f67acbab7d53..0701b15e15b43 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java @@ -20,7 +20,7 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.util.TestLogger; @@ -98,7 +98,7 @@ private static CompatibilityResult checkCompatibility(String enumSourceA, String ByteArrayOutputStream outBuffer = new ByteArrayOutputStream(); DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(outputViewStreamWrapper, snapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(outputViewStreamWrapper, snapshot); snapshotBytes = outBuffer.toByteArray(); } @@ -110,7 +110,7 @@ private static CompatibilityResult checkCompatibility(String enumSourceA, String ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes); DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) { - restoredSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot(inputViewStreamWrapper, classLoader2); + restoredSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(inputViewStreamWrapper, classLoader2); } EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME)); diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java index bdb3f8fb3e4d4..6b39a86bf05be 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java @@ -42,7 +42,7 @@ import org.apache.flink.api.common.operators.Keys.ExpressionKeys; import org.apache.flink.api.common.operators.Keys.IncompatibleKeysException; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.DateSerializer; import org.apache.flink.api.common.typeutils.base.DoubleSerializer; @@ -297,7 +297,7 @@ public void testReconfigureWithDifferentPojoType() throws Exception { TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer1.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); serializedConfig = out.toByteArray(); } @@ -306,7 +306,7 @@ public void testReconfigureWithDifferentPojoType() throws Exception { // read configuration again from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - pojoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + pojoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } @@ -333,7 +333,7 @@ public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); serializedConfig = out.toByteArray(); } @@ -346,7 +346,7 @@ public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - pojoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + pojoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } @@ -379,7 +379,7 @@ public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() thro TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); serializedConfig = out.toByteArray(); } @@ -389,7 +389,7 @@ public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() thro // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - pojoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + pojoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } @@ -437,7 +437,7 @@ public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Except TypeSerializerConfigSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot); serializedConfig = out.toByteArray(); } @@ -449,7 +449,7 @@ public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Except // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - pojoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + pojoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } @@ -552,7 +552,7 @@ public void testSerializerSerializationFailureResilience() throws Exception{ byte[] serializedConfig; try ( ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), config); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), config); serializedConfig = out.toByteArray(); } @@ -567,7 +567,7 @@ public void testSerializerSerializationFailureResilience() throws Exception{ PojoSerializer.PojoSerializerConfigSnapshot deserializedConfig; try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { deserializedConfig = (PojoSerializer.PojoSerializerConfigSnapshot) - TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), new ArtificialCNFExceptionThrowingClassLoader( Thread.currentThread().getContextClassLoader(), diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java index 89e9ec3f8c9e7..b1ff10d0155ab 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; @@ -63,7 +64,7 @@ public void testMigrationStrategyForRemovedAvroDependency() throws Exception { // read configuration again from bytes TypeSerializerConfigSnapshot kryoSerializerConfigSnapshot; try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) { - kryoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + kryoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } CompatibilityResult compatResult = kryoSerializerForA.ensureCompatibility(kryoSerializerConfigSnapshot); @@ -97,7 +98,7 @@ public void testMigrationStrategyWithDifferentKryoType() throws Exception { TypeSerializerConfigSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot); serializedConfig = out.toByteArray(); } @@ -105,7 +106,7 @@ public void testMigrationStrategyWithDifferentKryoType() throws Exception { // read configuration again from bytes try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - kryoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + kryoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } @@ -251,7 +252,7 @@ public void testMigrationStrategyForDifferentRegistrationOrder() throws Exceptio TypeSerializerConfigSnapshot kryoSerializerConfigSnapshot = kryoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot); serializedConfig = out.toByteArray(); } @@ -264,7 +265,7 @@ public void testMigrationStrategyForDifferentRegistrationOrder() throws Exceptio // read configuration from bytes try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { - kryoSerializerConfigSnapshot = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + kryoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala index 8be4debff353f..1fed5e425142a 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala @@ -21,7 +21,7 @@ package org.apache.flink.api.scala.typeutils import java.io._ import java.net.{URL, URLClassLoader} -import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializerSerializationUtil} +import org.apache.flink.api.common.typeutils.{CompatibilityResult, TypeSerializerConfigSnapshotSerializationUtil} import org.apache.flink.core.memory.{DataInputViewStreamWrapper, DataOutputViewStreamWrapper} import org.apache.flink.util.TestLogger import org.junit.rules.TemporaryFolder @@ -134,7 +134,7 @@ class EnumValueSerializerUpgradeTest extends TestLogger with JUnitSuiteLike { val baos = new ByteArrayOutputStream() val output = new DataOutputViewStreamWrapper(baos) - TypeSerializerSerializationUtil.writeSerializerConfigSnapshot(output, snapshot) + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(output, snapshot) output.close() baos.close() @@ -144,7 +144,7 @@ class EnumValueSerializerUpgradeTest extends TestLogger with JUnitSuiteLike { val classLoader2 = compileAndLoadEnum(tempFolder.newFolder(), s"$enumName.scala", enumSourceB) - val snapshot2 = TypeSerializerSerializationUtil.readSerializerConfigSnapshot( + val snapshot2 = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( input, classLoader2) val enum2 = instantiateEnum[Enumeration](classLoader2, enumName) From e09f91469fb6c86f5d2f05b78a9db3d9af8cce87 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 18 Jun 2018 16:24:08 +0200 Subject: [PATCH 4/6] [FLINK-9377] [core] (part 4) Introduce BackwardsCompatibleConfigSnapshot The BackwardsCompatibleConfigSnapshot is a wrapper, dummy config snapshot which wraps an actual config snapshot, as well as a pre-existing serializer instance. In previous versions, since the config snapshot wasn't a serializer factory but simply a container for serializer parameters, previous serializers didn't necessarily have config snapshots that are capable of correctly creating a correct corresponding restore serializer. In this case, since previous serializers still have serializers written in the checkpoint, the backwards compatible solution would be to wrap the written serializer and the config snapshot within the BackwardsCompatibleConfigSnapshot dummy. When attempting to restore the serializer, the wrapped serializer instance is returned instead of actually calling the restoreSerializer method of the wrapped config snapshot. --- .../BackwardsCompatibleConfigSnapshot.java | 102 ++++++++++++++++++ .../common/typeutils/CompatibilityUtil.java | 2 +- .../api/common/typeutils/TypeSerializer.java | 26 ++++- .../TypeSerializerSerializationUtil.java | 15 ++- .../common/typeutils/SerializerTestBase.java | 4 +- .../TypeSerializerSerializationUtilTest.java | 2 +- .../typeutils/base/EnumSerializerTest.java | 6 +- .../base/EnumSerializerUpgradeTest.java | 2 +- .../typeutils/runtime/PojoSerializerTest.java | 12 +-- .../kryo/KryoSerializerCompatibilityTest.java | 6 +- .../BackwardsCompatibleAvroSerializer.java | 6 +- ...BackwardsCompatibleAvroSerializerTest.java | 6 +- .../state/OperatorStateBackendTest.java | 2 +- .../TupleSerializerCompatibilityTest.scala | 3 +- .../typeutils/EnumValueSerializerTest.scala | 2 +- .../EnumValueSerializerUpgradeTest.scala | 2 +- 16 files changed, 166 insertions(+), 32 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/BackwardsCompatibleConfigSnapshot.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/BackwardsCompatibleConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/BackwardsCompatibleConfigSnapshot.java new file mode 100644 index 0000000000000..f3cef6c9b0769 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/BackwardsCompatibleConfigSnapshot.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; + +/** + * A utility {@link TypeSerializerConfigSnapshot} that is used for backwards compatibility purposes. + * + *

If this placeholder config snapshot is provided to a new serializer for compatibility checks, the wrapped + * config snapshot will be "unwrapped" and be provided instead. + * + * @see TypeSerializer#internalEnsureCompatibility(TypeSerializerConfigSnapshot) + * + * @param the data type that the wrapped serializer instance serializes. + */ +@Internal +public class BackwardsCompatibleConfigSnapshot extends TypeSerializerConfigSnapshot { + + private TypeSerializerConfigSnapshot wrappedConfigSnapshot; + + private TypeSerializer serializerInstance; + + public BackwardsCompatibleConfigSnapshot( + TypeSerializerConfigSnapshot wrappedConfigSnapshot, + TypeSerializer serializerInstance) { + + this.wrappedConfigSnapshot = Preconditions.checkNotNull(wrappedConfigSnapshot); + this.serializerInstance = Preconditions.checkNotNull(serializerInstance); + } + + @Override + public void write(DataOutputView out) throws IOException { + throw new UnsupportedOperationException( + "This is a dummy config snapshot used only for backwards compatibility."); + } + + @Override + public void read(DataInputView in) throws IOException { + throw new UnsupportedOperationException( + "This is a dummy config snapshot used only for backwards compatibility."); + } + + @Override + public int getVersion() { + throw new UnsupportedOperationException( + "This is a dummy config snapshot used only for backwards compatibility."); + } + + @Override + public TypeSerializer restoreSerializer() { + return serializerInstance; + } + + public TypeSerializerConfigSnapshot getWrappedConfigSnapshot() { + return wrappedConfigSnapshot; + } + + @Override + public int hashCode() { + int result = wrappedConfigSnapshot.hashCode(); + result = 31 * result + serializerInstance.hashCode(); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + BackwardsCompatibleConfigSnapshot that = (BackwardsCompatibleConfigSnapshot) o; + + return that.wrappedConfigSnapshot.equals(wrappedConfigSnapshot) + && that.serializerInstance.equals(serializerInstance); + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java index cff464943b934..0cfc03e6075da 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompatibilityUtil.java @@ -55,7 +55,7 @@ public static CompatibilityResult resolveCompatibilityResult( TypeSerializer newSerializer) { if (precedingSerializerConfigSnapshot != null) { - return newSerializer.ensureCompatibility(precedingSerializerConfigSnapshot); + return newSerializer.internalEnsureCompatibility(precedingSerializerConfigSnapshot); } else { // if the configuration snapshot of the preceding serializer cannot be provided, // we can only simply assume that the new serializer is compatible diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java index 017e09ebb3811..8eab786387f96 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializer.java @@ -18,6 +18,7 @@ package org.apache.flink.api.common.typeutils; +import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -215,5 +216,28 @@ public abstract class TypeSerializer implements Serializable { * * @return the determined compatibility result (cannot be {@code null}). */ - public abstract CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot); + protected abstract CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot); + + /** + * Public-facing method for serializer compatibility checks. Restored configuration snapshots should + * be provided via this method. + * + *

Before passing the configuration snapshot to the actual + * {@link #ensureCompatibility(TypeSerializerConfigSnapshot)} method, the configuration snapshot is checked + * to see if it is a dummy {@link BackwardsCompatibleConfigSnapshot}. If so, then the actual wrapped + * configuration snapshot is extracted and used instead. + * + * @param configSnapshot configuration snapshot of a preceding serializer for the same managed state + * + * @return the determined compatibility result (cannot be {@code null}). + */ + @Internal + public final CompatibilityResult internalEnsureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + if (configSnapshot instanceof BackwardsCompatibleConfigSnapshot) { + return ensureCompatibility( + ((BackwardsCompatibleConfigSnapshot) configSnapshot).getWrappedConfigSnapshot()); + } else { + return ensureCompatibility(configSnapshot); + } + } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java index f6258836b33fb..d02a02c0ac6c2 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java @@ -202,7 +202,7 @@ public static List, TypeSerializerConfigSnapshot>> read new ArrayList<>(numSerializersAndConfigSnapshots); TypeSerializer serializer; - TypeSerializerConfigSnapshot configSnapshot; + TypeSerializerConfigSnapshot configSnapshot; try ( ByteArrayInputStreamWithPos bufferWithPos = new ByteArrayInputStreamWithPos(buffer); DataInputViewStreamWrapper bufferWrapper = new DataInputViewStreamWrapper(bufferWithPos)) { @@ -213,10 +213,17 @@ public static List, TypeSerializerConfigSnapshot>> read serializer = tryReadSerializer(bufferWrapper, userCodeClassLoader, true); bufferWithPos.setPosition(offsets[i * 2 + 1]); - configSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(bufferWrapper, userCodeClassLoader); - serializersAndConfigSnapshots.add( - new Tuple2, TypeSerializerConfigSnapshot>(serializer, configSnapshot)); + // the config snapshot is replaced with a dummy one, which wraps + // the actual config snapshot and the deserialized serializer. + // this is for backwards compatibility reasons, since before Flink 1.6, some serializers + // do not return config snapshots that can be used as a factory for themselves. + configSnapshot = new BackwardsCompatibleConfigSnapshot<>( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( + bufferWrapper, userCodeClassLoader), + serializer); + + serializersAndConfigSnapshots.add(new Tuple2<>(serializer, configSnapshot)); } } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java index a2844d9cfe603..9aea9d52f61d0 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java @@ -120,11 +120,11 @@ public void testSnapshotConfigurationAndReconfigure() throws Exception { new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - CompatibilityResult strategy = getSerializer().ensureCompatibility(restoredConfig); + CompatibilityResult strategy = getSerializer().internalEnsureCompatibility(restoredConfig); assertFalse(strategy.isRequiresMigration()); // also verify that the serializer's reconfigure implementation detects incompatibility - strategy = getSerializer().ensureCompatibility(new TestIncompatibleSerializerConfigSnapshot<>()); + strategy = getSerializer().internalEnsureCompatibility(new TestIncompatibleSerializerConfigSnapshot<>()); assertTrue(strategy.isRequiresMigration()); } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java index 7f1e0f61574ad..da00fe7699f39 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtilTest.java @@ -416,7 +416,7 @@ public TypeSerializerConfigSnapshot snapshotConfiguration() { @Override public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - return IntSerializer.INSTANCE.ensureCompatibility(configSnapshot); + return IntSerializer.INSTANCE.internalEnsureCompatibility(configSnapshot); } @Override diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java index de564869d36a5..0001c9ac39026 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerTest.java @@ -72,7 +72,7 @@ public void testReconfiguration() { assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility - CompatibilityResult compatResult = serializer.ensureCompatibility( + CompatibilityResult compatResult = serializer.internalEnsureCompatibility( new EnumSerializer.EnumSerializerConfigSnapshot<>(PublicEnum.class, mockPreviousOrder)); assertFalse(compatResult.isRequiresMigration()); @@ -106,7 +106,7 @@ public void testConfigurationSnapshotSerialization() throws Exception { new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - CompatibilityResult compatResult = serializer.ensureCompatibility(restoredConfig); + CompatibilityResult compatResult = serializer.internalEnsureCompatibility(restoredConfig); assertFalse(compatResult.isRequiresMigration()); assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); @@ -161,7 +161,7 @@ public void testSerializeReconfiguredEnumSerializer() throws Exception { assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility - CompatibilityResult compatResult = serializer.ensureCompatibility( + CompatibilityResult compatResult = serializer.internalEnsureCompatibility( new EnumSerializer.EnumSerializerConfigSnapshot<>(PublicEnum.class, mockPreviousOrder)); assertFalse(compatResult.isRequiresMigration()); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java index 0701b15e15b43..09aeeb8ca590f 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/base/EnumSerializerUpgradeTest.java @@ -114,7 +114,7 @@ private static CompatibilityResult checkCompatibility(String enumSourceA, String } EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME)); - return enumSerializer2.ensureCompatibility(restoredSnapshot); + return enumSerializer2.internalEnsureCompatibility(restoredSnapshot); } private static ClassLoader compileAndLoadEnum(File root, String filename, String source) throws IOException { diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java index 6b39a86bf05be..09e3c26ca8055 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java @@ -310,7 +310,7 @@ public void testReconfigureWithDifferentPojoType() throws Exception { new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - CompatibilityResult compatResult = pojoSerializer2.ensureCompatibility(pojoSerializerConfigSnapshot); + CompatibilityResult compatResult = pojoSerializer2.internalEnsureCompatibility(pojoSerializerConfigSnapshot); assertTrue(compatResult.isRequiresMigration()); } @@ -350,7 +350,7 @@ public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - CompatibilityResult compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot); + CompatibilityResult compatResult = pojoSerializer.internalEnsureCompatibility(pojoSerializerConfigSnapshot); assertTrue(!compatResult.isRequiresMigration()); // reconfigure - check reconfiguration result and that registration ids remains the same @@ -394,7 +394,7 @@ public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() thro } // reconfigure - check reconfiguration result and that subclass serializer cache is repopulated - CompatibilityResult compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot); + CompatibilityResult compatResult = pojoSerializer.internalEnsureCompatibility(pojoSerializerConfigSnapshot); assertFalse(compatResult.isRequiresMigration()); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); @@ -456,7 +456,7 @@ public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Except // reconfigure - check reconfiguration result and that // 1) subclass serializer cache is repopulated // 2) registrations also contain the now registered subclasses - CompatibilityResult compatResult = pojoSerializer.ensureCompatibility(pojoSerializerConfigSnapshot); + CompatibilityResult compatResult = pojoSerializer.internalEnsureCompatibility(pojoSerializerConfigSnapshot); assertFalse(compatResult.isRequiresMigration()); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); @@ -533,7 +533,7 @@ public void testReconfigureWithDifferentFieldOrder() throws Exception { new HashMap, Tuple2, TypeSerializerConfigSnapshot>>()); // empty; irrelevant for this test // reconfigure - check reconfiguration result and that fields are reordered to the previous order - CompatibilityResult compatResult = pojoSerializer.ensureCompatibility(mockPreviousConfigSnapshot); + CompatibilityResult compatResult = pojoSerializer.internalEnsureCompatibility(mockPreviousConfigSnapshot); assertFalse(compatResult.isRequiresMigration()); int i = 0; for (Field field : mockOriginalFieldOrder) { @@ -574,7 +574,7 @@ public void testSerializerSerializationFailureResilience() throws Exception{ cnfThrowingClassnames)); } - Assert.assertFalse(pojoSerializer.ensureCompatibility(deserializedConfig).isRequiresMigration()); + Assert.assertFalse(pojoSerializer.internalEnsureCompatibility(deserializedConfig).isRequiresMigration()); verifyPojoSerializerConfigSnapshotWithSerializerSerializationFailure(config, deserializedConfig); } diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java index b1ff10d0155ab..222c32a8d860e 100644 --- a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerCompatibilityTest.java @@ -67,7 +67,7 @@ public void testMigrationStrategyForRemovedAvroDependency() throws Exception { kryoSerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - CompatibilityResult compatResult = kryoSerializerForA.ensureCompatibility(kryoSerializerConfigSnapshot); + CompatibilityResult compatResult = kryoSerializerForA.internalEnsureCompatibility(kryoSerializerConfigSnapshot); assertFalse(compatResult.isRequiresMigration()); } @@ -110,7 +110,7 @@ public void testMigrationStrategyWithDifferentKryoType() throws Exception { new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } - CompatibilityResult compatResult = kryoSerializerForB.ensureCompatibility(kryoSerializerConfigSnapshot); + CompatibilityResult compatResult = kryoSerializerForB.internalEnsureCompatibility(kryoSerializerConfigSnapshot); assertTrue(compatResult.isRequiresMigration()); } @@ -270,7 +270,7 @@ public void testMigrationStrategyForDifferentRegistrationOrder() throws Exceptio } // reconfigure - check reconfiguration result and that registration id remains the same - CompatibilityResult compatResult = kryoSerializer.ensureCompatibility(kryoSerializerConfigSnapshot); + CompatibilityResult compatResult = kryoSerializer.internalEnsureCompatibility(kryoSerializerConfigSnapshot); assertFalse(compatResult.isRequiresMigration()); assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId()); assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId()); diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java index addd4b1295423..573f6940cae19 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializer.java @@ -186,7 +186,7 @@ public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot pojoSerializer = (TypeSerializer) typeInfo.createPojoSerializer(new ExecutionConfig()); this.serializer = pojoSerializer; - return serializer.ensureCompatibility(configSnapshot); + return serializer.internalEnsureCompatibility(configSnapshot); } else if (configSnapshot instanceof KryoRegistrationSerializerConfigSnapshot) { // force-kryo old case common previous case // we create a new Kryo Serializer with a blank execution config. // registrations are anyways picked up from the snapshot. serializer = new KryoSerializer<>(type, new ExecutionConfig()); - return serializer.ensureCompatibility(configSnapshot); + return serializer.internalEnsureCompatibility(configSnapshot); } else { // completely incompatible type, needs migration diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializerTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializerTest.java index f64163656fa78..29a8adce94a6f 100644 --- a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializerTest.java +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/BackwardsCompatibleAvroSerializerTest.java @@ -102,10 +102,10 @@ public void testCompatibilityWithPojoSerializer() throws Exception { validateDeserialization(serializer); // sanity check for the test: check that a PoJoSerializer and the original serializer work together - assertFalse(serializer.ensureCompatibility(configSnapshot).isRequiresMigration()); + assertFalse(serializer.internalEnsureCompatibility(configSnapshot).isRequiresMigration()); final TypeSerializer newSerializer = new AvroTypeInfo<>(User.class, true).createSerializer(new ExecutionConfig()); - assertFalse(newSerializer.ensureCompatibility(configSnapshot).isRequiresMigration()); + assertFalse(newSerializer.internalEnsureCompatibility(configSnapshot).isRequiresMigration()); // deserialize the data and make sure this still works validateDeserialization(newSerializer); @@ -113,7 +113,7 @@ public void testCompatibilityWithPojoSerializer() throws Exception { TypeSerializerConfigSnapshot nextSnapshot = newSerializer.snapshotConfiguration(); final TypeSerializer nextSerializer = new AvroTypeInfo<>(User.class, true).createSerializer(new ExecutionConfig()); - assertFalse(nextSerializer.ensureCompatibility(nextSnapshot).isRequiresMigration()); + assertFalse(nextSerializer.internalEnsureCompatibility(nextSnapshot).isRequiresMigration()); // deserialize the data and make sure this still works validateDeserialization(nextSerializer); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java index bbf7fd5676382..114a8304a0c5c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/OperatorStateBackendTest.java @@ -357,7 +357,7 @@ public TypeSerializerConfigSnapshot snapshotConfiguration() { @Override public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - return IntSerializer.INSTANCE.ensureCompatibility(configSnapshot); + return IntSerializer.INSTANCE.internalEnsureCompatibility(configSnapshot); } } diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerCompatibilityTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerCompatibilityTest.scala index 7d594fb2b16c3..29991dba9f4a5 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerCompatibilityTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerCompatibilityTest.scala @@ -59,7 +59,8 @@ class TupleSerializerCompatibilityTest { val currentSerializer = createTypeInformation[TestCaseClass] .createSerializer(new ExecutionConfig()) - assertFalse(currentSerializer.ensureCompatibility(oldConfigSnapshot).isRequiresMigration) + assertFalse(currentSerializer + .internalEnsureCompatibility(oldConfigSnapshot).isRequiresMigration) // test old data serialization is.close() diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerTest.scala index f89e9bc473860..be7cfd48413ca 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerTest.scala @@ -35,7 +35,7 @@ class EnumValueSerializerTest extends TestLogger with JUnitSuiteLike { val snapshot = enumSerializer.snapshotConfiguration() - assertFalse(enumSerializer.ensureCompatibility(snapshot).isRequiresMigration) + assertFalse(enumSerializer.internalEnsureCompatibility(snapshot).isRequiresMigration) } } diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala index 1fed5e425142a..ff2b1247b747e 100644 --- a/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala +++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/typeutils/EnumValueSerializerUpgradeTest.scala @@ -150,7 +150,7 @@ class EnumValueSerializerUpgradeTest extends TestLogger with JUnitSuiteLike { val enum2 = instantiateEnum[Enumeration](classLoader2, enumName) val enumValueSerializer2 = new EnumValueSerializer(enum2) - enumValueSerializer2.ensureCompatibility(snapshot2) + enumValueSerializer2.internalEnsureCompatibility(snapshot2) } } From da84665a9b101a803f7446210afc34bbd4a71703 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 2 Jul 2018 11:45:20 +0800 Subject: [PATCH 5/6] [FLINK-9377] [core] (part 5) Remove serializers from checkpoint state meta infos This commit officially removes the behaviour of writing serializers in the state meta info of keyed state, operator state, and timers state. This affects the serialization formats of the KeyedBackendSerializationProxy, OperatorBackendSerializationProxy, and InternalTimerServiceSerializationProxy, and therefore their versions are all upticked. --- .../state/DefaultOperatorStateBackend.java | 23 ++- .../state/KeyedBackendSerializationProxy.java | 58 +++--- ...endStateMetaInfoSnapshotReaderWriters.java | 109 +++++++++--- .../OperatorBackendSerializationProxy.java | 6 +- ...endStateMetaInfoSnapshotReaderWriters.java | 165 ++++++++++++++---- ...gisteredBroadcastBackendStateMetaInfo.java | 26 +-- .../RegisteredKeyedBackendStateMetaInfo.java | 53 ++---- ...egisteredOperatorBackendStateMetaInfo.java | 13 +- .../state/heap/HeapKeyedStateBackend.java | 4 +- .../state/SerializationProxiesTest.java | 13 -- .../state/RocksDBKeyedStateBackend.java | 12 +- .../operators/HeapInternalTimerService.java | 22 ++- ...nternalTimerServiceSerializationProxy.java | 7 +- .../api/operators/InternalTimersSnapshot.java | 47 ++--- .../InternalTimersSnapshotReaderWriters.java | 95 +++++++--- 15 files changed, 405 insertions(+), 248 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java index 58083b8c87612..e0b40c86a28d2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/DefaultOperatorStateBackend.java @@ -499,8 +499,10 @@ public void restore(Collection restoreSnapshots) throws Exc // Recreate all PartitionableListStates from the meta info for (RegisteredOperatorBackendStateMetaInfo.Snapshot restoredMetaInfo : restoredOperatorMetaInfoSnapshots) { - if (restoredMetaInfo.getPartitionStateSerializer() == null || - restoredMetaInfo.getPartitionStateSerializer() instanceof UnloadableDummyTypeSerializer) { + TypeSerializer restoredPartitionStateSerializer = + restoredMetaInfo.getPartitionStateSerializerConfigSnapshot().restoreSerializer(); + + if (restoredPartitionStateSerializer instanceof UnloadableDummyTypeSerializer) { // must fail now if the previous serializer cannot be restored because there is no serializer // capable of reading previous state @@ -521,7 +523,7 @@ public void restore(Collection restoreSnapshots) throws Exc listState = new PartitionableListState<>( new RegisteredOperatorBackendStateMetaInfo<>( restoredMetaInfo.getName(), - restoredMetaInfo.getPartitionStateSerializer(), + restoredPartitionStateSerializer, restoredMetaInfo.getAssignmentMode())); registeredOperatorStates.put(listState.getStateMetaInfo().getName(), listState); @@ -536,9 +538,14 @@ public void restore(Collection restoreSnapshots) throws Exc for (RegisteredBroadcastBackendStateMetaInfo.Snapshot restoredMetaInfo : restoredBroadcastMetaInfoSnapshots) { - if (restoredMetaInfo.getKeySerializer() == null || restoredMetaInfo.getValueSerializer() == null || - restoredMetaInfo.getKeySerializer() instanceof UnloadableDummyTypeSerializer || - restoredMetaInfo.getValueSerializer() instanceof UnloadableDummyTypeSerializer) { + TypeSerializer restoredKeySerializer = + restoredMetaInfo.getKeySerializerConfigSnapshot().restoreSerializer(); + + TypeSerializer restoredValueSerializer = + restoredMetaInfo.getValueSerializerConfigSnapshot().restoreSerializer(); + + if (restoredKeySerializer instanceof UnloadableDummyTypeSerializer || + restoredValueSerializer instanceof UnloadableDummyTypeSerializer) { // must fail now if the previous serializer cannot be restored because there is no serializer // capable of reading previous state @@ -560,8 +567,8 @@ public void restore(Collection restoreSnapshots) throws Exc new RegisteredBroadcastBackendStateMetaInfo<>( restoredMetaInfo.getName(), restoredMetaInfo.getAssignmentMode(), - restoredMetaInfo.getKeySerializer(), - restoredMetaInfo.getValueSerializer())); + restoredKeySerializer, + restoredValueSerializer)); registeredBroadcastStates.put(broadcastState.getStateMetaInfo().getName(), broadcastState); } else { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java index f32e646e6efd9..10d681e3f2f70 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendSerializationProxy.java @@ -18,8 +18,10 @@ package org.apache.flink.runtime.state; +import org.apache.flink.api.common.typeutils.BackwardsCompatibleConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.java.tuple.Tuple2; @@ -30,7 +32,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -39,7 +40,7 @@ */ public class KeyedBackendSerializationProxy extends VersionedIOReadableWritable { - public static final int VERSION = 4; + public static final int VERSION = 5; //TODO allow for more (user defined) compression formats + backwards compatibility story. /** This specifies if we use a compressed format write the key-groups */ @@ -48,8 +49,7 @@ public class KeyedBackendSerializationProxy extends VersionedIOReadableWritab /** This specifies whether or not to use dummy {@link UnloadableDummyTypeSerializer} when serializers cannot be read. */ private boolean isSerializerPresenceRequired; - private TypeSerializer keySerializer; - private TypeSerializerConfigSnapshot keySerializerConfigSnapshot; + private TypeSerializerConfigSnapshot keySerializerConfigSnapshot; private List> stateMetaInfoSnapshots; @@ -67,7 +67,7 @@ public KeyedBackendSerializationProxy( this.usingKeyGroupCompression = compression; - this.keySerializer = Preconditions.checkNotNull(keySerializer); + Preconditions.checkNotNull(keySerializer); this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializer.snapshotConfiguration()); Preconditions.checkNotNull(stateMetaInfoSnapshots); @@ -79,10 +79,6 @@ public KeyedBackendSerializationProxy( return stateMetaInfoSnapshots; } - public TypeSerializer getKeySerializer() { - return keySerializer; - } - public TypeSerializerConfigSnapshot getKeySerializerConfigSnapshot() { return keySerializerConfigSnapshot; } @@ -98,8 +94,8 @@ public int getVersion() { @Override public int[] getCompatibleVersions() { - // we are compatible with version 3 (Flink 1.3.x) and version 1 & 2 (Flink 1.2.x) - return new int[] {VERSION, 3, 2, 1}; + // we are compatible with version 4 (Flink 1.5.x), version 3 (Flink 1.3.x+) and version 1 & 2 (Flink 1.2.x) + return new int[] {VERSION, 4, 3, 2, 1}; } @Override @@ -109,11 +105,7 @@ public void write(DataOutputView out) throws IOException { // write the compression format used to write each key-group out.writeBoolean(usingKeyGroupCompression); - // write in a way to be fault tolerant of read failures when deserializing the key serializer - TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience( - out, - Collections.singletonList( - new Tuple2, TypeSerializerConfigSnapshot>(keySerializer, keySerializerConfigSnapshot))); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot(out, keySerializerConfigSnapshot); // write individual registered keyed state metainfos out.writeShort(stateMetaInfoSnapshots.size()); @@ -137,19 +129,23 @@ public void read(DataInputView in) throws IOException { usingKeyGroupCompression = false; } - // only starting from version 3, we have the key serializer and its config snapshot written - if (readVersion >= 3) { + if (readVersion >= 5) { + this.keySerializerConfigSnapshot = TypeSerializerConfigSnapshotSerializationUtil + .readSerializerConfigSnapshot(in, userCodeClassLoader); + } else if (readVersion >= 3) { + // versions 3 and 4 still write serializers in checkpoints Tuple2, TypeSerializerConfigSnapshot> keySerializerAndConfig = - TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader).get(0); - this.keySerializer = (TypeSerializer) keySerializerAndConfig.f0; - this.keySerializerConfigSnapshot = keySerializerAndConfig.f1; + TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader).get(0); + + this.keySerializerConfigSnapshot = new BackwardsCompatibleConfigSnapshot( + keySerializerAndConfig.f1, + (TypeSerializer) keySerializerAndConfig.f0); } else { - this.keySerializer = TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true); - this.keySerializerConfigSnapshot = null; - } + // read through the serializer + TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true); - if (isSerializerPresenceRequired) { - checkSerializerPresence(keySerializer); + // there is not config snapshot in either older versions (<= 2) + this.keySerializerConfigSnapshot = null; } int numKvStates = in.readShort(); @@ -160,17 +156,17 @@ public void read(DataInputView in) throws IOException { .readStateMetaInfo(in); if (isSerializerPresenceRequired) { - checkSerializerPresence(snapshot.getNamespaceSerializer()); - checkSerializerPresence(snapshot.getStateSerializer()); + checkSerializerPresence(snapshot.getNamespaceSerializerConfigSnapshot()); + checkSerializerPresence(snapshot.getStateSerializerConfigSnapshot()); } stateMetaInfoSnapshots.add(snapshot); } } - private void checkSerializerPresence(TypeSerializer serializer) throws IOException { - if (serializer instanceof UnloadableDummyTypeSerializer) { + private void checkSerializerPresence(TypeSerializerConfigSnapshot snapshot) throws IOException { + if (snapshot.restoreSerializer() instanceof UnloadableDummyTypeSerializer) { throw new IOException("Unable to restore keyed state, because a previous serializer" + - " of the keyed state is not present The serializer could have been removed from the classpath, " + + " of the keyed state is not present. The serializer could have been removed from the classpath, " + " or its implementation have changed and could not be loaded. This is a temporary restriction that will" + " be fixed in future versions."); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java index f99234180c90b..5277de7fdcfc9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/KeyedBackendStateMetaInfoSnapshotReaderWriters.java @@ -19,8 +19,10 @@ package org.apache.flink.runtime.state; import org.apache.flink.api.common.state.StateDescriptor; +import org.apache.flink.api.common.typeutils.BackwardsCompatibleConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; @@ -41,6 +43,9 @@ public class KeyedBackendStateMetaInfoSnapshotReaderWriters { // Writers // - v1: Flink 1.2.x // - v2: Flink 1.3.x + // - v3: Flink 1.4.x + // - v4: Flink 1.5.x + // - v5: Flink 1.6.x // ------------------------------------------------------------------------------- public static KeyedBackendStateMetaInfoWriter getWriterForVersion( @@ -52,9 +57,12 @@ public static KeyedBackendStateMetaInfoWriter getWriterForVersion( return new KeyedBackendStateMetaInfoWriterV1V2<>(stateMetaInfo); case 3: + case 4: + return new KeyedBackendStateMetaInfoWriterV3V4<>(stateMetaInfo); + // current version case KeyedBackendSerializationProxy.VERSION: - return new KeyedBackendStateMetaInfoWriterV3<>(stateMetaInfo); + return new KeyedBackendStateMetaInfoWriterV5<>(stateMetaInfo); default: // guard for future @@ -88,14 +96,16 @@ public void writeStateMetaInfo(DataOutputView out) throws IOException { out.writeInt(stateMetaInfo.getStateType().ordinal()); out.writeUTF(stateMetaInfo.getName()); - TypeSerializerSerializationUtil.writeSerializer(out, stateMetaInfo.getNamespaceSerializer()); - TypeSerializerSerializationUtil.writeSerializer(out, stateMetaInfo.getStateSerializer()); + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats + TypeSerializerSerializationUtil.writeSerializer(out, null); + TypeSerializerSerializationUtil.writeSerializer(out, null); } } - static class KeyedBackendStateMetaInfoWriterV3 extends AbstractKeyedBackendStateMetaInfoWriter { + static class KeyedBackendStateMetaInfoWriterV3V4 extends AbstractKeyedBackendStateMetaInfoWriter { - public KeyedBackendStateMetaInfoWriterV3(RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfo) { + public KeyedBackendStateMetaInfoWriterV3V4(RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfo) { super(stateMetaInfo); } @@ -107,11 +117,31 @@ public void writeStateMetaInfo(DataOutputView out) throws IOException { // write in a way that allows us to be fault-tolerant and skip blocks in the case of java serialization failures TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience( out, + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats Arrays.asList( - new Tuple2, TypeSerializerConfigSnapshot>( - stateMetaInfo.getNamespaceSerializer(), stateMetaInfo.getNamespaceSerializerConfigSnapshot()), - new Tuple2, TypeSerializerConfigSnapshot>( - stateMetaInfo.getStateSerializer(), stateMetaInfo.getStateSerializerConfigSnapshot()))); + new Tuple2<>( + null, stateMetaInfo.getNamespaceSerializerConfigSnapshot()), + new Tuple2<>( + null, stateMetaInfo.getStateSerializerConfigSnapshot()))); + } + } + + static class KeyedBackendStateMetaInfoWriterV5 extends AbstractKeyedBackendStateMetaInfoWriter { + + public KeyedBackendStateMetaInfoWriterV5(RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfo) { + super(stateMetaInfo); + } + + @Override + public void writeStateMetaInfo(DataOutputView out) throws IOException { + out.writeInt(stateMetaInfo.getStateType().ordinal()); + out.writeUTF(stateMetaInfo.getName()); + + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, stateMetaInfo.getNamespaceSerializerConfigSnapshot()); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, stateMetaInfo.getStateSerializerConfigSnapshot()); } } @@ -120,6 +150,9 @@ public void writeStateMetaInfo(DataOutputView out) throws IOException { // Readers // - v1: Flink 1.2.x // - v2: Flink 1.3.x + // - v3: Flink 1.4.x + // - v4: Flink 1.5.x + // - v5: Flink 1.6.x // ------------------------------------------------------------------------------- public static KeyedBackendStateMetaInfoReader getReaderForVersion( @@ -130,10 +163,13 @@ public static KeyedBackendStateMetaInfoReader getReaderForVersion( case 2: return new KeyedBackendStateMetaInfoReaderV1V2<>(userCodeClassLoader); - // current version case 3: + case 4: + return new KeyedBackendStateMetaInfoReaderV3V4<>(userCodeClassLoader); + + // current version case KeyedBackendSerializationProxy.VERSION: - return new KeyedBackendStateMetaInfoReaderV3<>(userCodeClassLoader); + return new KeyedBackendStateMetaInfoReaderV5<>(userCodeClassLoader); default: // guard for future @@ -170,21 +206,27 @@ public RegisteredKeyedBackendStateMetaInfo.Snapshot readStateMetaInfo(Data metaInfo.setStateType(StateDescriptor.Type.values()[in.readInt()]); metaInfo.setName(in.readUTF()); - metaInfo.setNamespaceSerializer(TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true)); - metaInfo.setStateSerializer(TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true)); + final TypeSerializerSerializationUtil.TypeSerializerSerializationProxy namespaceSerializerProxy = + new TypeSerializerSerializationUtil.TypeSerializerSerializationProxy<>(userCodeClassLoader); + namespaceSerializerProxy.read(in); - // older versions do not contain the configuration snapshot - metaInfo.setNamespaceSerializerConfigSnapshot(null); - metaInfo.setStateSerializerConfigSnapshot(null); + final TypeSerializerSerializationUtil.TypeSerializerSerializationProxy stateSerializerProxy = + new TypeSerializerSerializationUtil.TypeSerializerSerializationProxy<>(userCodeClassLoader); + stateSerializerProxy.read(in); + + // older versions do not contain the configuration snapshot; + // we deserialize the written serializers, and then simply take a snapshot of them now + metaInfo.setNamespaceSerializerConfigSnapshot(namespaceSerializerProxy.getTypeSerializer().snapshotConfiguration()); + metaInfo.setStateSerializerConfigSnapshot(stateSerializerProxy.getTypeSerializer().snapshotConfiguration()); return metaInfo; } } @SuppressWarnings("unchecked") - static class KeyedBackendStateMetaInfoReaderV3 extends AbstractKeyedBackendStateMetaInfoReader { + static class KeyedBackendStateMetaInfoReaderV3V4 extends AbstractKeyedBackendStateMetaInfoReader { - public KeyedBackendStateMetaInfoReaderV3(ClassLoader userCodeClassLoader) { + public KeyedBackendStateMetaInfoReaderV3V4(ClassLoader userCodeClassLoader) { super(userCodeClassLoader); } @@ -199,11 +241,34 @@ public RegisteredKeyedBackendStateMetaInfo.Snapshot readStateMetaInfo(Data List, TypeSerializerConfigSnapshot>> serializersAndConfigs = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader); - metaInfo.setNamespaceSerializer((TypeSerializer) serializersAndConfigs.get(0).f0); - metaInfo.setNamespaceSerializerConfigSnapshot(serializersAndConfigs.get(0).f1); + metaInfo.setNamespaceSerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(serializersAndConfigs.get(0).f1, serializersAndConfigs.get(0).f0)); + metaInfo.setStateSerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(serializersAndConfigs.get(1).f1, serializersAndConfigs.get(1).f0)); + + return metaInfo; + } + } + + static class KeyedBackendStateMetaInfoReaderV5 extends AbstractKeyedBackendStateMetaInfoReader { + + public KeyedBackendStateMetaInfoReaderV5(ClassLoader userCodeClassLoader) { + super(userCodeClassLoader); + } + + @SuppressWarnings("unchecked") + @Override + public RegisteredKeyedBackendStateMetaInfo.Snapshot readStateMetaInfo(DataInputView in) throws IOException { + RegisteredKeyedBackendStateMetaInfo.Snapshot metaInfo = + new RegisteredKeyedBackendStateMetaInfo.Snapshot<>(); + + metaInfo.setStateType(StateDescriptor.Type.values()[in.readInt()]); + metaInfo.setName(in.readUTF()); - metaInfo.setStateSerializer((TypeSerializer) serializersAndConfigs.get(1).f0); - metaInfo.setStateSerializerConfigSnapshot(serializersAndConfigs.get(1).f1); + metaInfo.setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshotSerializationUtil + .readSerializerConfigSnapshot(in, userCodeClassLoader)); + metaInfo.setStateSerializerConfigSnapshot(TypeSerializerConfigSnapshotSerializationUtil + .readSerializerConfigSnapshot(in, userCodeClassLoader)); return metaInfo; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java index e73f83a9e1d96..d1767d50e8e4d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendSerializationProxy.java @@ -33,7 +33,7 @@ */ public class OperatorBackendSerializationProxy extends VersionedIOReadableWritable { - public static final int VERSION = 3; + public static final int VERSION = 4; private List> operatorStateMetaInfoSnapshots; private List> broadcastStateMetaInfoSnapshots; @@ -62,8 +62,8 @@ public int getVersion() { @Override public int[] getCompatibleVersions() { - // we are compatible with version 3 (Flink 1.5.x), 2 (Flink 1.4.x, Flink 1.3.x) and version 1 (Flink 1.2.x) - return new int[] {VERSION, 2, 1}; + // we are compatible with version 4 (Flink 1.6.x), 3 (Flink 1.5.x), 2 (Flink 1.4.x, Flink 1.3.x) and version 1 (Flink 1.2.x) + return new int[] {VERSION, 3, 2, 1}; } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java index fafd5423e01c3..9a0329298de2d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java @@ -18,8 +18,10 @@ package org.apache.flink.runtime.state; +import org.apache.flink.api.common.typeutils.BackwardsCompatibleConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream; @@ -27,8 +29,6 @@ import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Arrays; @@ -41,13 +41,12 @@ */ public class OperatorBackendStateMetaInfoSnapshotReaderWriters { - private static final Logger LOG = LoggerFactory.getLogger(OperatorBackendStateMetaInfoSnapshotReaderWriters.class); - // ------------------------------------------------------------------------------- // Writers // - v1: Flink 1.2.x // - v2: Flink 1.3.x // - v3: Flink 1.5.x + // - v4: Flink 1.6.x // ------------------------------------------------------------------------------- public static OperatorBackendStateMetaInfoWriter getOperatorStateWriterForVersion( @@ -59,9 +58,12 @@ public static OperatorBackendStateMetaInfoWriter getOperatorStateWriterForVe // current version case 2: - case OperatorBackendSerializationProxy.VERSION: - return new OperatorBackendStateMetaInfoWriterV2<>(stateMetaInfo); + case 3: + return new OperatorBackendStateMetaInfoWriterV2V3<>(stateMetaInfo); + // current version + case OperatorBackendSerializationProxy.VERSION: + return new OperatorBackendStateMetaInfoWriterV4<>(stateMetaInfo); default: // guard for future throw new IllegalStateException( @@ -74,9 +76,12 @@ public static BroadcastStateMetaInfoWriter getBroadcastStateWriterForVers final RegisteredBroadcastBackendStateMetaInfo.Snapshot broadcastStateMetaInfo) { switch (version) { + case 3: + return new BroadcastStateMetaInfoWriterV3<>(broadcastStateMetaInfo); + // current version case OperatorBackendSerializationProxy.VERSION: - return new BroadcastStateMetaInfoWriterV3<>(broadcastStateMetaInfo); + return new BroadcastStateMetaInfoWriterV4<>(broadcastStateMetaInfo); default: // guard for future @@ -123,13 +128,16 @@ public OperatorBackendStateMetaInfoWriterV1(RegisteredOperatorBackendStateMetaIn public void writeOperatorStateMetaInfo(DataOutputView out) throws IOException { out.writeUTF(stateMetaInfo.getName()); out.writeByte(stateMetaInfo.getAssignmentMode().ordinal()); - TypeSerializerSerializationUtil.writeSerializer(out, stateMetaInfo.getPartitionStateSerializer()); + + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats + TypeSerializerSerializationUtil.writeSerializer(out, null); } } - public static class OperatorBackendStateMetaInfoWriterV2 extends AbstractOperatorBackendStateMetaInfoWriter { + public static class OperatorBackendStateMetaInfoWriterV2V3 extends AbstractOperatorBackendStateMetaInfoWriter { - public OperatorBackendStateMetaInfoWriterV2(RegisteredOperatorBackendStateMetaInfo.Snapshot stateMetaInfo) { + public OperatorBackendStateMetaInfoWriterV2V3(RegisteredOperatorBackendStateMetaInfo.Snapshot stateMetaInfo) { super(stateMetaInfo); } @@ -141,12 +149,31 @@ public void writeOperatorStateMetaInfo(DataOutputView out) throws IOException { // write in a way that allows us to be fault-tolerant and skip blocks in the case of java serialization failures TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience( out, - Collections.singletonList(new Tuple2, TypeSerializerConfigSnapshot>( - stateMetaInfo.getPartitionStateSerializer(), + Collections.singletonList(new Tuple2<>( + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats + null, stateMetaInfo.getPartitionStateSerializerConfigSnapshot()))); } } + public static class OperatorBackendStateMetaInfoWriterV4 extends AbstractOperatorBackendStateMetaInfoWriter { + + public OperatorBackendStateMetaInfoWriterV4(RegisteredOperatorBackendStateMetaInfo.Snapshot stateMetaInfo) { + super(stateMetaInfo); + } + + @Override + public void writeOperatorStateMetaInfo(DataOutputView out) throws IOException { + out.writeUTF(stateMetaInfo.getName()); + out.writeByte(stateMetaInfo.getAssignmentMode().ordinal()); + + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, + stateMetaInfo.getPartitionStateSerializerConfigSnapshot()); + } + } + public static class BroadcastStateMetaInfoWriterV3 extends AbstractBroadcastStateMetaInfoWriter { public BroadcastStateMetaInfoWriterV3( @@ -162,13 +189,15 @@ public void writeBroadcastStateMetaInfo(final DataOutputView out) throws IOExcep // write in a way that allows us to be fault-tolerant and skip blocks in the case of java serialization failures TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience( out, + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats Arrays.asList( Tuple2.of( - broadcastStateMetaInfo.getKeySerializer(), + null, broadcastStateMetaInfo.getKeySerializerConfigSnapshot() ), Tuple2.of( - broadcastStateMetaInfo.getValueSerializer(), + null, broadcastStateMetaInfo.getValueSerializerConfigSnapshot() ) ) @@ -176,11 +205,34 @@ public void writeBroadcastStateMetaInfo(final DataOutputView out) throws IOExcep } } + public static class BroadcastStateMetaInfoWriterV4 extends AbstractBroadcastStateMetaInfoWriter { + + public BroadcastStateMetaInfoWriterV4( + final RegisteredBroadcastBackendStateMetaInfo.Snapshot broadcastStateMetaInfo) { + super(broadcastStateMetaInfo); + } + + @Override + public void writeBroadcastStateMetaInfo(final DataOutputView out) throws IOException { + out.writeUTF(broadcastStateMetaInfo.getName()); + out.writeByte(broadcastStateMetaInfo.getAssignmentMode().ordinal()); + + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, + broadcastStateMetaInfo.getKeySerializerConfigSnapshot()); + + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, + broadcastStateMetaInfo.getValueSerializerConfigSnapshot()); + } + } + // ------------------------------------------------------------------------------- // Readers // - v1: Flink 1.2.x // - v2: Flink 1.3.x // - v3: Flink 1.5.x + // - v4: Flink 1.6.x // ------------------------------------------------------------------------------- public static OperatorBackendStateMetaInfoReader getOperatorStateReaderForVersion( @@ -190,10 +242,13 @@ public static OperatorBackendStateMetaInfoReader getOperatorStateReaderFo case 1: return new OperatorBackendStateMetaInfoReaderV1<>(userCodeClassLoader); - // version 2 and version 3 (current) case 2: + case 3: + return new OperatorBackendStateMetaInfoReaderV2V3<>(userCodeClassLoader); + + // current version case OperatorBackendSerializationProxy.VERSION: - return new OperatorBackendStateMetaInfoReaderV2<>(userCodeClassLoader); + return new OperatorBackendStateMetaInfoReaderV4<>(userCodeClassLoader); default: // guard for future @@ -206,9 +261,12 @@ public static BroadcastStateMetaInfoReader getBroadcastStateReaderF int version, ClassLoader userCodeClassLoader) { switch (version) { + case 3: + return new BroadcastStateMetaInfoReaderV3<>(userCodeClassLoader); + // current version case OperatorBackendSerializationProxy.VERSION: - return new BroadcastStateMetaInfoReaderV3<>(userCodeClassLoader); + return new BroadcastStateMetaInfoReaderV4<>(userCodeClassLoader); default: // guard for future @@ -262,30 +320,31 @@ public RegisteredOperatorBackendStateMetaInfo.Snapshot readOperatorStateMetaI DataInputViewStream dis = new DataInputViewStream(in); ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader(); + TypeSerializer stateSerializer; try ( InstantiationUtil.FailureTolerantObjectInputStream ois = new InstantiationUtil.FailureTolerantObjectInputStream(dis, userCodeClassLoader)) { Thread.currentThread().setContextClassLoader(userCodeClassLoader); - TypeSerializer stateSerializer = (TypeSerializer) ois.readObject(); - stateMetaInfo.setPartitionStateSerializer(stateSerializer); + stateSerializer = (TypeSerializer) ois.readObject(); } catch (ClassNotFoundException exception) { throw new IOException(exception); } finally { Thread.currentThread().setContextClassLoader(previousClassLoader); } - // old versions do not contain the partition state serializer's configuration snapshot - stateMetaInfo.setPartitionStateSerializerConfigSnapshot(null); + // old versions do not contain the partition state serializer's configuration snapshot; + // we deserialize the written serializers, and then simply take a snapshot of them now + stateMetaInfo.setPartitionStateSerializerConfigSnapshot(stateSerializer.snapshotConfiguration()); return stateMetaInfo; } } @SuppressWarnings("unchecked") - public static class OperatorBackendStateMetaInfoReaderV2 extends AbstractOperatorBackendStateMetaInfoReader { + public static class OperatorBackendStateMetaInfoReaderV2V3 extends AbstractOperatorBackendStateMetaInfoReader { - public OperatorBackendStateMetaInfoReaderV2(ClassLoader userCodeClassLoader) { + public OperatorBackendStateMetaInfoReaderV2V3(ClassLoader userCodeClassLoader) { super(userCodeClassLoader); } @@ -300,13 +359,35 @@ public RegisteredOperatorBackendStateMetaInfo.Snapshot readOperatorStateMetaI Tuple2, TypeSerializerConfigSnapshot> stateSerializerAndConfig = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader).get(0); - stateMetaInfo.setPartitionStateSerializer((TypeSerializer) stateSerializerAndConfig.f0); - stateMetaInfo.setPartitionStateSerializerConfigSnapshot(stateSerializerAndConfig.f1); + stateMetaInfo.setPartitionStateSerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(stateSerializerAndConfig.f1, stateSerializerAndConfig.f0)); + + return stateMetaInfo; + } + } + + public static class OperatorBackendStateMetaInfoReaderV4 extends AbstractOperatorBackendStateMetaInfoReader { + + public OperatorBackendStateMetaInfoReaderV4(ClassLoader userCodeClassLoader) { + super(userCodeClassLoader); + } + + @Override + public RegisteredOperatorBackendStateMetaInfo.Snapshot readOperatorStateMetaInfo(DataInputView in) throws IOException { + RegisteredOperatorBackendStateMetaInfo.Snapshot stateMetaInfo = + new RegisteredOperatorBackendStateMetaInfo.Snapshot<>(); + + stateMetaInfo.setName(in.readUTF()); + stateMetaInfo.setAssignmentMode(OperatorStateHandle.Mode.values()[in.readByte()]); + + stateMetaInfo.setPartitionStateSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(in, userCodeClassLoader)); return stateMetaInfo; } } + @SuppressWarnings("unchecked") public static class BroadcastStateMetaInfoReaderV3 extends AbstractBroadcastStateMetaInfoReader { public BroadcastStateMetaInfoReaderV3(final ClassLoader userCodeClassLoader) { @@ -327,11 +408,35 @@ public RegisteredBroadcastBackendStateMetaInfo.Snapshot readBroadcastState Tuple2, TypeSerializerConfigSnapshot> keySerializerAndConfig = serializers.get(0); Tuple2, TypeSerializerConfigSnapshot> valueSerializerAndConfig = serializers.get(1); - stateMetaInfo.setKeySerializer((TypeSerializer) keySerializerAndConfig.f0); - stateMetaInfo.setKeySerializerConfigSnapshot(keySerializerAndConfig.f1); - - stateMetaInfo.setValueSerializer((TypeSerializer) valueSerializerAndConfig.f0); - stateMetaInfo.setValueSerializerConfigSnapshot(valueSerializerAndConfig.f1); + stateMetaInfo.setKeySerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(keySerializerAndConfig.f1, keySerializerAndConfig.f0)); + + stateMetaInfo.setValueSerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(valueSerializerAndConfig.f1, valueSerializerAndConfig.f0)); + + return stateMetaInfo; + } + } + + public static class BroadcastStateMetaInfoReaderV4 extends AbstractBroadcastStateMetaInfoReader { + + public BroadcastStateMetaInfoReaderV4(final ClassLoader userCodeClassLoader) { + super(userCodeClassLoader); + } + + @Override + public RegisteredBroadcastBackendStateMetaInfo.Snapshot readBroadcastStateMetaInfo(DataInputView in) throws IOException { + RegisteredBroadcastBackendStateMetaInfo.Snapshot stateMetaInfo = + new RegisteredBroadcastBackendStateMetaInfo.Snapshot<>(); + + stateMetaInfo.setName(in.readUTF()); + stateMetaInfo.setAssignmentMode(OperatorStateHandle.Mode.values()[in.readByte()]); + + stateMetaInfo.setKeySerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(in, userCodeClassLoader)); + + stateMetaInfo.setValueSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(in, userCodeClassLoader)); return stateMetaInfo; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastBackendStateMetaInfo.java index 7204cd3856346..a491559840c7a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastBackendStateMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredBroadcastBackendStateMetaInfo.java @@ -140,8 +140,6 @@ public static class Snapshot { private String name; private OperatorStateHandle.Mode assignmentMode; - private TypeSerializer keySerializer; - private TypeSerializer valueSerializer; private TypeSerializerConfigSnapshot keySerializerConfigSnapshot; private TypeSerializerConfigSnapshot valueSerializerConfigSnapshot; @@ -158,8 +156,8 @@ private Snapshot( this.name = Preconditions.checkNotNull(name); this.assignmentMode = Preconditions.checkNotNull(assignmentMode); - this.keySerializer = Preconditions.checkNotNull(keySerializer); - this.valueSerializer = Preconditions.checkNotNull(valueSerializer); + Preconditions.checkNotNull(keySerializer); + Preconditions.checkNotNull(valueSerializer); this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializerConfigSnapshot); this.valueSerializerConfigSnapshot = Preconditions.checkNotNull(valueSerializerConfigSnapshot); } @@ -180,22 +178,6 @@ void setAssignmentMode(OperatorStateHandle.Mode mode) { this.assignmentMode = mode; } - public TypeSerializer getKeySerializer() { - return keySerializer; - } - - void setKeySerializer(TypeSerializer serializer) { - this.keySerializer = serializer; - } - - public TypeSerializer getValueSerializer() { - return valueSerializer; - } - - void setValueSerializer(TypeSerializer serializer) { - this.valueSerializer = serializer; - } - public TypeSerializerConfigSnapshot getKeySerializerConfigSnapshot() { return keySerializerConfigSnapshot; } @@ -227,8 +209,6 @@ public boolean equals(Object obj) { return name.equals(snapshot.getName()) && assignmentMode.ordinal() == snapshot.getAssignmentMode().ordinal() - && Objects.equals(keySerializer, snapshot.getKeySerializer()) - && Objects.equals(valueSerializer, snapshot.getValueSerializer()) && keySerializerConfigSnapshot.equals(snapshot.getKeySerializerConfigSnapshot()) && valueSerializerConfigSnapshot.equals(snapshot.getValueSerializerConfigSnapshot()); } @@ -237,8 +217,6 @@ public boolean equals(Object obj) { public int hashCode() { int result = name.hashCode(); result = 31 * result + assignmentMode.hashCode(); - result = 31 * result + ((keySerializer != null) ? keySerializer.hashCode() : 0); - result = 31 * result + ((valueSerializer != null) ? valueSerializer.hashCode() : 0); result = 31 * result + keySerializerConfigSnapshot.hashCode(); result = 31 * result + valueSerializerConfigSnapshot.hashCode(); return result; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java index 3f7a8d6f8247e..fbf620b60cd42 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredKeyedBackendStateMetaInfo.java @@ -77,9 +77,7 @@ public Snapshot snapshot() { stateType, name, namespaceSerializer.duplicate(), - stateSerializer.duplicate(), - namespaceSerializer.snapshotConfiguration(), - stateSerializer.snapshotConfiguration()); + stateSerializer.duplicate()); } @Override @@ -132,10 +130,8 @@ public static class Snapshot { private StateDescriptor.Type stateType; private String name; - private TypeSerializer namespaceSerializer; - private TypeSerializer stateSerializer; - private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot; - private TypeSerializerConfigSnapshot stateSerializerConfigSnapshot; + private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot; + private TypeSerializerConfigSnapshot stateSerializerConfigSnapshot; /** Empty constructor used when restoring the state meta info snapshot. */ Snapshot() {} @@ -144,16 +140,15 @@ private Snapshot( StateDescriptor.Type stateType, String name, TypeSerializer namespaceSerializer, - TypeSerializer stateSerializer, - TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot, - TypeSerializerConfigSnapshot stateSerializerConfigSnapshot) { + TypeSerializer stateSerializer) { this.stateType = Preconditions.checkNotNull(stateType); this.name = Preconditions.checkNotNull(name); - this.namespaceSerializer = Preconditions.checkNotNull(namespaceSerializer); - this.stateSerializer = Preconditions.checkNotNull(stateSerializer); - this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializerConfigSnapshot); - this.stateSerializerConfigSnapshot = Preconditions.checkNotNull(stateSerializerConfigSnapshot); + + Preconditions.checkNotNull(namespaceSerializer); + Preconditions.checkNotNull(stateSerializer); + this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializer.snapshotConfiguration()); + this.stateSerializerConfigSnapshot = Preconditions.checkNotNull(stateSerializer.snapshotConfiguration()); } public StateDescriptor.Type getStateType() { @@ -172,35 +167,19 @@ void setName(String name) { this.name = name; } - public TypeSerializer getNamespaceSerializer() { - return namespaceSerializer; - } - - void setNamespaceSerializer(TypeSerializer namespaceSerializer) { - this.namespaceSerializer = namespaceSerializer; - } - - public TypeSerializer getStateSerializer() { - return stateSerializer; - } - - void setStateSerializer(TypeSerializer stateSerializer) { - this.stateSerializer = stateSerializer; - } - - public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() { + public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() { return namespaceSerializerConfigSnapshot; } - void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) { + void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) { this.namespaceSerializerConfigSnapshot = namespaceSerializerConfigSnapshot; } - public TypeSerializerConfigSnapshot getStateSerializerConfigSnapshot() { + public TypeSerializerConfigSnapshot getStateSerializerConfigSnapshot() { return stateSerializerConfigSnapshot; } - void setStateSerializerConfigSnapshot(TypeSerializerConfigSnapshot stateSerializerConfigSnapshot) { + void setStateSerializerConfigSnapshot(TypeSerializerConfigSnapshot stateSerializerConfigSnapshot) { this.stateSerializerConfigSnapshot = stateSerializerConfigSnapshot; } @@ -225,9 +204,7 @@ public boolean equals(Object o) { } // need to check for nulls because serializer and config snapshots may be null on restore - return Objects.equals(getStateSerializer(), that.getStateSerializer()) - && Objects.equals(getNamespaceSerializer(), that.getNamespaceSerializer()) - && Objects.equals(getNamespaceSerializerConfigSnapshot(), that.getNamespaceSerializerConfigSnapshot()) + return Objects.equals(getNamespaceSerializerConfigSnapshot(), that.getNamespaceSerializerConfigSnapshot()) && Objects.equals(getStateSerializerConfigSnapshot(), that.getStateSerializerConfigSnapshot()); } @@ -236,8 +213,6 @@ public int hashCode() { // need to check for nulls because serializer and config snapshots may be null on restore int result = getName().hashCode(); result = 31 * result + getStateType().hashCode(); - result = 31 * result + (getNamespaceSerializer() != null ? getNamespaceSerializer().hashCode() : 0); - result = 31 * result + (getStateSerializer() != null ? getStateSerializer().hashCode() : 0); result = 31 * result + (getNamespaceSerializerConfigSnapshot() != null ? getNamespaceSerializerConfigSnapshot().hashCode() : 0); result = 31 * result + (getStateSerializerConfigSnapshot() != null ? getStateSerializerConfigSnapshot().hashCode() : 0); return result; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java index a9adc8d1e3a01..9d8428f8a3b2e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/RegisteredOperatorBackendStateMetaInfo.java @@ -133,7 +133,6 @@ public static class Snapshot { private String name; private OperatorStateHandle.Mode assignmentMode; - private TypeSerializer partitionStateSerializer; private TypeSerializerConfigSnapshot partitionStateSerializerConfigSnapshot; /** Empty constructor used when restoring the state meta info snapshot. */ @@ -147,7 +146,7 @@ private Snapshot( this.name = Preconditions.checkNotNull(name); this.assignmentMode = Preconditions.checkNotNull(assignmentMode); - this.partitionStateSerializer = Preconditions.checkNotNull(partitionStateSerializer); + Preconditions.checkNotNull(partitionStateSerializer); this.partitionStateSerializerConfigSnapshot = Preconditions.checkNotNull(partitionStateSerializerConfigSnapshot); } @@ -167,14 +166,6 @@ void setAssignmentMode(OperatorStateHandle.Mode assignmentMode) { this.assignmentMode = assignmentMode; } - public TypeSerializer getPartitionStateSerializer() { - return partitionStateSerializer; - } - - void setPartitionStateSerializer(TypeSerializer partitionStateSerializer) { - this.partitionStateSerializer = partitionStateSerializer; - } - public TypeSerializerConfigSnapshot getPartitionStateSerializerConfigSnapshot() { return partitionStateSerializerConfigSnapshot; } @@ -202,7 +193,6 @@ public boolean equals(Object obj) { // need to check for nulls because serializer and config snapshots may be null on restore return name.equals(snapshot.getName()) && assignmentMode.equals(snapshot.getAssignmentMode()) - && Objects.equals(partitionStateSerializer, snapshot.getPartitionStateSerializer()) && Objects.equals(partitionStateSerializerConfigSnapshot, snapshot.getPartitionStateSerializerConfigSnapshot()); } @@ -211,7 +201,6 @@ public int hashCode() { // need to check for nulls because serializer and config snapshots may be null on restore int result = getName().hashCode(); result = 31 * result + getAssignmentMode().hashCode(); - result = 31 * result + (getPartitionStateSerializer() != null ? getPartitionStateSerializer().hashCode() : 0); result = 31 * result + (getPartitionStateSerializerConfigSnapshot() != null ? getPartitionStateSerializerConfigSnapshot().hashCode() : 0); return result; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index 87c3b57df541c..491e8cfc807ad 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -318,8 +318,8 @@ private void restorePartitionedState(Collection state) throws new RegisteredKeyedBackendStateMetaInfo<>( restoredMetaInfo.getStateType(), restoredMetaInfo.getName(), - restoredMetaInfo.getNamespaceSerializer(), - restoredMetaInfo.getStateSerializer()); + restoredMetaInfo.getNamespaceSerializerConfigSnapshot().restoreSerializer(), + restoredMetaInfo.getStateSerializerConfigSnapshot().restoreSerializer()); stateTable = snapshotStrategy.newStateTable(registeredKeyedBackendStateMetaInfo); stateTables.put(restoredMetaInfo.getName(), stateTable); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java index 3f78097f11cd0..30dad1a3aa4b3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/SerializationProxiesTest.java @@ -20,7 +20,6 @@ import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer; import org.apache.flink.api.common.typeutils.base.DoubleSerializer; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.common.typeutils.base.LongSerializer; @@ -74,7 +73,6 @@ public void testKeyedBackendSerializationProxyRoundtrip() throws Exception { } Assert.assertEquals(true, serializationProxy.isUsingKeyGroupCompression()); - Assert.assertEquals(keySerializer, serializationProxy.getKeySerializer()); Assert.assertEquals(keySerializer.snapshotConfiguration(), serializationProxy.getKeySerializerConfigSnapshot()); Assert.assertEquals(stateMetaInfoList, serializationProxy.getStateMetaInfoSnapshots()); } @@ -123,12 +121,9 @@ public void testKeyedBackendSerializationProxyRoundtripWithSerializerSerializati } Assert.assertEquals(true, serializationProxy.isUsingKeyGroupCompression()); - Assert.assertTrue(serializationProxy.getKeySerializer() instanceof UnloadableDummyTypeSerializer); Assert.assertEquals(keySerializer.snapshotConfiguration(), serializationProxy.getKeySerializerConfigSnapshot()); for (RegisteredKeyedBackendStateMetaInfo.Snapshot meta : serializationProxy.getStateMetaInfoSnapshots()) { - Assert.assertTrue(meta.getNamespaceSerializer() instanceof UnloadableDummyTypeSerializer); - Assert.assertTrue(meta.getStateSerializer() instanceof UnloadableDummyTypeSerializer); Assert.assertEquals(namespaceSerializer.snapshotConfiguration(), meta.getNamespaceSerializerConfigSnapshot()); Assert.assertEquals(stateSerializer.snapshotConfiguration(), meta.getStateSerializerConfigSnapshot()); } @@ -195,8 +190,6 @@ public void testKeyedStateMetaInfoReadSerializerFailureResilience() throws Excep } Assert.assertEquals(name, metaInfo.getName()); - Assert.assertTrue(metaInfo.getNamespaceSerializer() instanceof UnloadableDummyTypeSerializer); - Assert.assertTrue(metaInfo.getStateSerializer() instanceof UnloadableDummyTypeSerializer); Assert.assertEquals(namespaceSerializer.snapshotConfiguration(), metaInfo.getNamespaceSerializerConfigSnapshot()); Assert.assertEquals(stateSerializer.snapshotConfiguration(), metaInfo.getStateSerializerConfigSnapshot()); } @@ -271,7 +264,6 @@ public void testOperatorStateMetaInfoSerialization() throws Exception { Assert.assertEquals(name, metaInfo.getName()); Assert.assertEquals(OperatorStateHandle.Mode.UNION, metaInfo.getAssignmentMode()); - Assert.assertEquals(stateSerializer, metaInfo.getPartitionStateSerializer()); } @Test @@ -302,8 +294,6 @@ public void testBroadcastStateMetaInfoSerialization() throws Exception { Assert.assertEquals(name, metaInfo.getName()); Assert.assertEquals(OperatorStateHandle.Mode.BROADCAST, metaInfo.getAssignmentMode()); - Assert.assertEquals(keySerializer, metaInfo.getKeySerializer()); - Assert.assertEquals(valueSerializer, metaInfo.getValueSerializer()); } @Test @@ -339,7 +329,6 @@ public void testOperatorStateMetaInfoReadSerializerFailureResilience() throws Ex } Assert.assertEquals(name, metaInfo.getName()); - Assert.assertTrue(metaInfo.getPartitionStateSerializer() instanceof UnloadableDummyTypeSerializer); Assert.assertEquals(stateSerializer.snapshotConfiguration(), metaInfo.getPartitionStateSerializerConfigSnapshot()); } @@ -377,9 +366,7 @@ public void testBroadcastStateMetaInfoReadSerializerFailureResilience() throws E } Assert.assertEquals(broadcastName, broadcastMetaInfo.getName()); - Assert.assertTrue(broadcastMetaInfo.getKeySerializer() instanceof UnloadableDummyTypeSerializer); Assert.assertEquals(keySerializer.snapshotConfiguration(), broadcastMetaInfo.getKeySerializerConfigSnapshot()); - Assert.assertTrue(broadcastMetaInfo.getValueSerializer() instanceof UnloadableDummyTypeSerializer); Assert.assertEquals(valueSerializer.snapshotConfiguration(), broadcastMetaInfo.getValueSerializerConfigSnapshot()); } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index a7b8b4cb4580d..1d00267892f9b 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -643,8 +643,8 @@ private void restoreKVStateMetaData() throws IOException, StateMigrationExceptio new RegisteredKeyedBackendStateMetaInfo<>( restoredMetaInfo.getStateType(), restoredMetaInfo.getName(), - restoredMetaInfo.getNamespaceSerializer(), - restoredMetaInfo.getStateSerializer()); + restoredMetaInfo.getNamespaceSerializerConfigSnapshot().restoreSerializer(), + restoredMetaInfo.getStateSerializerConfigSnapshot().restoreSerializer()); rocksDBKeyedStateBackend.restoredKvStateMetaInfos.put(restoredMetaInfo.getName(), restoredMetaInfo); @@ -949,8 +949,8 @@ private ColumnFamilyHandle getOrRegisterColumnFamilyHandle( new RegisteredKeyedBackendStateMetaInfo<>( stateMetaInfoSnapshot.getStateType(), stateMetaInfoSnapshot.getName(), - stateMetaInfoSnapshot.getNamespaceSerializer(), - stateMetaInfoSnapshot.getStateSerializer()); + stateMetaInfoSnapshot.getNamespaceSerializerConfigSnapshot().restoreSerializer(), + stateMetaInfoSnapshot.getStateSerializerConfigSnapshot().restoreSerializer()); registeredStateMetaInfoEntry = new Tuple2<>( @@ -1082,8 +1082,8 @@ private void restoreLocalStateIntoFullInstance( new RegisteredKeyedBackendStateMetaInfo<>( stateMetaInfoSnapshot.getStateType(), stateMetaInfoSnapshot.getName(), - stateMetaInfoSnapshot.getNamespaceSerializer(), - stateMetaInfoSnapshot.getStateSerializer()); + stateMetaInfoSnapshot.getNamespaceSerializerConfigSnapshot().restoreSerializer(), + stateMetaInfoSnapshot.getStateSerializerConfigSnapshot().restoreSerializer()); stateBackend.kvStateInformation.put( stateMetaInfoSnapshot.getName(), diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java index be85748bd8625..5d447e04f2c47 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java @@ -253,9 +253,7 @@ public void advanceWatermark(long time) throws Exception { public InternalTimersSnapshot snapshotTimersForKeyGroup(int keyGroupIdx) { return new InternalTimersSnapshot<>( keySerializer, - keySerializer.snapshotConfiguration(), namespaceSerializer, - namespaceSerializer.snapshotConfiguration(), eventTimeTimersQueue.getTimersForKeyGroup(keyGroupIdx), processingTimeTimersQueue.getTimersForKeyGroup(keyGroupIdx)); } @@ -271,13 +269,18 @@ public InternalTimersSnapshot snapshotTimersForKeyGroup(int keyGroupIdx) { public void restoreTimersForKeyGroup(InternalTimersSnapshot restoredSnapshot, int keyGroupIdx) { this.restoredTimersSnapshot = (InternalTimersSnapshot) restoredSnapshot; - if (areSnapshotSerializersIncompatible(restoredSnapshot)) { + TypeSerializer restoredKeyDeserializer = + restoredTimersSnapshot.getKeySerializerConfigSnapshot().restoreSerializer(); + TypeSerializer restoredNamespaceDeserializer = + restoredTimersSnapshot.getNamespaceSerializerConfigSnapshot().restoreSerializer(); + + if (areSnapshotSerializersIncompatible(restoredKeyDeserializer, restoredNamespaceDeserializer)) { throw new IllegalArgumentException("Tried to restore timers " + "for the same service with different serializers."); } - this.keyDeserializer = restoredTimersSnapshot.getKeySerializer(); - this.namespaceDeserializer = restoredTimersSnapshot.getNamespaceSerializer(); + this.keyDeserializer = restoredKeyDeserializer; + this.namespaceDeserializer = restoredNamespaceDeserializer; checkArgument(localKeyGroupRange.contains(keyGroupIdx), "Key Group " + keyGroupIdx + " does not belong to the local range."); @@ -332,8 +335,11 @@ List>> getProcessingTimeTimersPerKeyGroup() { return processingTimeTimersQueue.getTimersByKeyGroup(); } - private boolean areSnapshotSerializersIncompatible(InternalTimersSnapshot restoredSnapshot) { - return (this.keyDeserializer != null && !this.keyDeserializer.equals(restoredSnapshot.getKeySerializer())) || - (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(restoredSnapshot.getNamespaceSerializer())); + private boolean areSnapshotSerializersIncompatible( + TypeSerializer restoredKeySerializer, + TypeSerializer restoredNamespaceSerializer) { + + return (this.keyDeserializer != null && !this.keyDeserializer.equals(restoredKeySerializer)) || + (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(restoredNamespaceSerializer)); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java index efa93d3e266a0..900a81cc0d24f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java @@ -36,7 +36,7 @@ @Internal public class InternalTimerServiceSerializationProxy extends PostVersionedIOReadableWritable { - public static final int VERSION = 1; + public static final int VERSION = 2; /** The key-group timer services to write / read. */ private Map> timerServices; @@ -88,6 +88,11 @@ public int getVersion() { return VERSION; } + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + @Override public void write(DataOutputView out) throws IOException { super.write(out); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java index ddaee6b2e7a44..cb2e0a163233c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java @@ -32,10 +32,8 @@ */ public class InternalTimersSnapshot { - private TypeSerializer keySerializer; - private TypeSerializerConfigSnapshot keySerializerConfigSnapshot; - private TypeSerializer namespaceSerializer; - private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot; + private TypeSerializerConfigSnapshot keySerializerConfigSnapshot; + private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot; private Set> eventTimeTimers; private Set> processingTimeTimers; @@ -46,49 +44,31 @@ public InternalTimersSnapshot() {} /** Constructor to use when snapshotting the timers. */ public InternalTimersSnapshot( TypeSerializer keySerializer, - TypeSerializerConfigSnapshot keySerializerConfigSnapshot, TypeSerializer namespaceSerializer, - TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot, @Nullable Set> eventTimeTimers, @Nullable Set> processingTimeTimers) { - this.keySerializer = Preconditions.checkNotNull(keySerializer); - this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializerConfigSnapshot); - this.namespaceSerializer = Preconditions.checkNotNull(namespaceSerializer); - this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializerConfigSnapshot); + Preconditions.checkNotNull(keySerializer); + this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializer.snapshotConfiguration()); + Preconditions.checkNotNull(namespaceSerializer); + this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializer.snapshotConfiguration()); this.eventTimeTimers = eventTimeTimers; this.processingTimeTimers = processingTimeTimers; } - public TypeSerializer getKeySerializer() { - return keySerializer; - } - - public void setKeySerializer(TypeSerializer keySerializer) { - this.keySerializer = keySerializer; - } - public TypeSerializerConfigSnapshot getKeySerializerConfigSnapshot() { return keySerializerConfigSnapshot; } - public void setKeySerializerConfigSnapshot(TypeSerializerConfigSnapshot keySerializerConfigSnapshot) { + public void setKeySerializerConfigSnapshot(TypeSerializerConfigSnapshot keySerializerConfigSnapshot) { this.keySerializerConfigSnapshot = keySerializerConfigSnapshot; } - public TypeSerializer getNamespaceSerializer() { - return namespaceSerializer; - } - - public void setNamespaceSerializer(TypeSerializer namespaceSerializer) { - this.namespaceSerializer = namespaceSerializer; - } - public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() { return namespaceSerializerConfigSnapshot; } - public void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) { + public void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) { this.namespaceSerializerConfigSnapshot = namespaceSerializerConfigSnapshot; } @@ -108,6 +88,17 @@ public void setProcessingTimeTimers(Set> processingTimeTimer this.processingTimeTimers = processingTimeTimers; } + public TimerHeapInternalTimer.TimerSerializer createTimerSerializer() { + Preconditions.checkState(keySerializerConfigSnapshot != null && namespaceSerializerConfigSnapshot != null, + "Key / namespace serializer config snapshots are null; if the timer snapshot" + + " was restored from a checkpoint, the serializer config snapshots must be restored first before" + + " attempting to create the timer serializer."); + + return new TimerHeapInternalTimer.TimerSerializer<>( + keySerializerConfigSnapshot.restoreSerializer(), + namespaceSerializerConfigSnapshot.restoreSerializer()); + } + @Override public boolean equals(Object obj) { return super.equals(obj); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java index 05b77a7143cb0..b3c1c3359ce35 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java @@ -19,8 +19,10 @@ package org.apache.flink.streaming.api.operators; import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.BackwardsCompatibleConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshotSerializationUtil; import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream; @@ -49,7 +51,8 @@ public class InternalTimersSnapshotReaderWriters { // ------------------------------------------------------------------------------- // Writers // - pre-versioned: Flink 1.4.0 - // - v1: Flink 1.4.1 + // - v1: Flink 1.4.1, 1.5.x + // - v2: Flink 1.6.x // ------------------------------------------------------------------------------- public static InternalTimersSnapshotWriter getWriterForVersion(int version, InternalTimersSnapshot timersSnapshot) { @@ -58,9 +61,13 @@ public static InternalTimersSnapshotWriter getWriterForVersion(int versio case NO_VERSION: return new InternalTimersSnapshotWriterPreVersioned<>(timersSnapshot); - case InternalTimerServiceSerializationProxy.VERSION: + case 1: return new InternalTimersSnapshotWriterV1<>(timersSnapshot); + // current version: + case InternalTimerServiceSerializationProxy.VERSION: + return new InternalTimersSnapshotWriterV2<>(timersSnapshot); + default: // guard for future throw new IllegalStateException( @@ -96,9 +103,7 @@ public AbstractInternalTimersSnapshotWriter(InternalTimersSnapshot timersS public final void writeTimersSnapshot(DataOutputView out) throws IOException { writeKeyAndNamespaceSerializers(out); - TimerHeapInternalTimer.TimerSerializer timerSerializer = new TimerHeapInternalTimer.TimerSerializer<>( - timersSnapshot.getKeySerializer(), - timersSnapshot.getNamespaceSerializer()); + TimerHeapInternalTimer.TimerSerializer timerSerializer = timersSnapshot.createTimerSerializer(); // write the event time timers Set> eventTimers = timersSnapshot.getEventTimeTimers(); @@ -134,8 +139,10 @@ public InternalTimersSnapshotWriterPreVersioned(InternalTimersSnapshot tim protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException { // the pre-versioned format only serializes the serializers, without their configuration snapshots try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) { - InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer()); - InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer()); + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats + InstantiationUtil.serializeObject(stream, null); + InstantiationUtil.serializeObject(stream, null); out.write(stream.getBuf(), 0, stream.getPosition()); } @@ -153,16 +160,34 @@ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOExce // write key / namespace serializers, and their configuration snapshots TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience( out, + // state meta info snapshots no longer contain serializers, so we use null just as a placeholder; + // this is maintained here to keep track of previous versions' serialization formats Arrays.asList( - Tuple2.of(timersSnapshot.getKeySerializer(), timersSnapshot.getKeySerializerConfigSnapshot()), - Tuple2.of(timersSnapshot.getNamespaceSerializer(), timersSnapshot.getNamespaceSerializerConfigSnapshot()))); + Tuple2.of(null, timersSnapshot.getKeySerializerConfigSnapshot()), + Tuple2.of(null, timersSnapshot.getNamespaceSerializerConfigSnapshot()))); + } + } + + private static class InternalTimersSnapshotWriterV2 extends AbstractInternalTimersSnapshotWriter { + + public InternalTimersSnapshotWriterV2(InternalTimersSnapshot timersSnapshot) { + super(timersSnapshot); + } + + @Override + protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException { + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, timersSnapshot.getKeySerializerConfigSnapshot()); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshot( + out, timersSnapshot.getNamespaceSerializerConfigSnapshot()); } } // ------------------------------------------------------------------------------- // Readers // - pre-versioned: Flink 1.4.0 - // - v1: Flink 1.4.1 + // - v1: Flink 1.4.1, 1.5.x + // - v2: Flink 1.6.x // ------------------------------------------------------------------------------- public static InternalTimersSnapshotReader getReaderForVersion( @@ -172,9 +197,13 @@ public static InternalTimersSnapshotReader getReaderForVersion( case NO_VERSION: return new InternalTimersSnapshotReaderPreVersioned<>(userCodeClassLoader); - case InternalTimerServiceSerializationProxy.VERSION: + case 1: return new InternalTimersSnapshotReaderV1<>(userCodeClassLoader); + // current version + case InternalTimerServiceSerializationProxy.VERSION: + return new InternalTimersSnapshotReaderV2<>(userCodeClassLoader); + default: // guard for future throw new IllegalStateException( @@ -215,10 +244,7 @@ public final InternalTimersSnapshot readTimersSnapshot(DataInputView in) t restoreKeyAndNamespaceSerializers(restoredTimersSnapshot, in); - TimerHeapInternalTimer.TimerSerializer timerSerializer = - new TimerHeapInternalTimer.TimerSerializer<>( - restoredTimersSnapshot.getKeySerializer(), - restoredTimersSnapshot.getNamespaceSerializer()); + TimerHeapInternalTimer.TimerSerializer timerSerializer = restoredTimersSnapshot.createTimerSerializer(); // read the event time timers int sizeOfEventTimeTimers = in.readInt(); @@ -260,8 +286,14 @@ protected void restoreKeyAndNamespaceSerializers( DataInputViewStream dis = new DataInputViewStream(in); try { - restoredTimersSnapshot.setKeySerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true)); - restoredTimersSnapshot.setNamespaceSerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true)); + // Flink 1.4.0 and below did not write the serializer config snapshots for timers; + // we deserialize the written serializers, and then simply take a snapshot of them now + + TypeSerializer keySerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true); + restoredTimersSnapshot.setKeySerializerConfigSnapshot(keySerializer.snapshotConfiguration()); + + TypeSerializer namespaceSerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true); + restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot(namespaceSerializer.snapshotConfiguration()); } catch (ClassNotFoundException exception) { throw new IOException(exception); } @@ -283,10 +315,31 @@ protected void restoreKeyAndNamespaceSerializers( List, TypeSerializerConfigSnapshot>> serializersAndConfigs = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader); - restoredTimersSnapshot.setKeySerializer((TypeSerializer) serializersAndConfigs.get(0).f0); - restoredTimersSnapshot.setKeySerializerConfigSnapshot(serializersAndConfigs.get(0).f1); - restoredTimersSnapshot.setNamespaceSerializer((TypeSerializer) serializersAndConfigs.get(1).f0); - restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot(serializersAndConfigs.get(1).f1); + restoredTimersSnapshot.setKeySerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(serializersAndConfigs.get(0).f1, serializersAndConfigs.get(0).f0)); + + restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot( + new BackwardsCompatibleConfigSnapshot(serializersAndConfigs.get(1).f1, serializersAndConfigs.get(1).f0)); + } + } + + private static class InternalTimersSnapshotReaderV2 extends AbstractInternalTimersSnapshotReader { + + public InternalTimersSnapshotReaderV2(ClassLoader userCodeClassLoader) { + super(userCodeClassLoader); + } + + @SuppressWarnings("unchecked") + @Override + protected void restoreKeyAndNamespaceSerializers( + InternalTimersSnapshot restoredTimersSnapshot, + DataInputView in) throws IOException { + + restoredTimersSnapshot.setKeySerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(in, userCodeClassLoader)); + + restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot( + TypeSerializerConfigSnapshotSerializationUtil.readSerializerConfigSnapshot(in, userCodeClassLoader)); } } } From 4e9ee72c3176e821188df91bca13cdd1bb239280 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 18 Jun 2018 16:24:35 +0200 Subject: [PATCH 6/6] [FLINK-9377] [core] (part 6) Properly implement restoreSerializer for simple composite serializer config snapshots --- .../typeutils/CompositeTypeSerializer.java | 101 ++++++++++++++++++ ...CompositeTypeSerializerConfigSnapshot.java | 74 ++++++++++--- .../CollectionSerializerConfigSnapshot.java | 23 +++- .../base/GenericArraySerializer.java | 36 +++---- .../GenericArraySerializerConfigSnapshot.java | 20 +++- .../common/typeutils/base/ListSerializer.java | 42 +++----- .../base/ListSerializerConfigSnapshot.java | 51 +++++++++ .../common/typeutils/base/MapSerializer.java | 51 +++------ .../base/MapSerializerConfigSnapshot.java | 20 +++- .../typeutils/runtime/EitherSerializer.java | 48 ++------- .../EitherSerializerConfigSnapshot.java | 20 +++- .../java/typeutils/runtime/RowSerializer.java | 64 ++++------- .../typeutils/runtime/TupleSerializer.java | 26 +++-- .../runtime/TupleSerializerBase.java | 62 ++--------- .../TupleSerializerConfigSnapshot.java | 22 +++- .../java/org/apache/flink/cep/nfa/NFA.java | 61 +++++------ .../apache/flink/cep/nfa/SharedBuffer.java | 74 ++++++------- .../flink/cep/nfa/sharedbuffer/Lockable.java | 37 ++----- .../LockableSerializerConfigSnapshot.java | 5 + .../ListViewSerializerConfigSnapshot.java | 7 ++ .../MapViewSerializerConfigSnapshot.java | 7 ++ .../table/dataview/ListViewSerializer.scala | 40 ++----- .../table/dataview/MapViewSerializer.scala | 45 ++------ .../table/runtime/types/CRowSerializer.scala | 20 +++- .../runtime/state/ArrayListSerializer.java | 40 +++---- .../ArrayListSerializerConfigSnapshot.java | 48 +++++++++ .../ScalaEitherSerializerConfigSnapshot.java | 8 ++ .../ScalaOptionSerializerConfigSnapshot.java | 18 +++- .../ScalaTrySerializerConfigSnapshot.java | 20 +++- .../TraversableSerializerConfigSnapshot.java | 6 ++ .../scala/codegen/TypeInformationGen.scala | 9 -- .../org/apache/flink/api/scala/package.scala | 6 -- .../scala/typeutils/CaseClassSerializer.scala | 13 +-- .../scala/typeutils/EitherSerializer.scala | 50 ++------- .../scala/typeutils/OptionSerializer.scala | 61 +++++------ .../typeutils/TraversableSerializer.scala | 2 +- .../api/scala/typeutils/TrySerializer.scala | 76 ++++++------- .../api/scala/typeutils/TryTypeInfo.scala | 15 ++- .../api/datastream/CoGroupedStreams.java | 62 ++++++----- .../sink/TwoPhaseCommitSinkFunction.java | 66 ++++++------ .../streamrecord/StreamElementSerializer.java | 69 +++++------- 41 files changed, 810 insertions(+), 715 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializer.java create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerConfigSnapshot.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerConfigSnapshot.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializer.java new file mode 100644 index 0000000000000..c650e882aa612 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializer.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.Preconditions; + +/** + * A base class for simple composite type serializers which consists of multiple nested + * type serializers. The constructor takes a pre-instantiated {@link CompositeTypeSerializerConfigSnapshot} + * instance, which is cached and returned on each {@link #snapshotConfiguration()} call. + * When compatibility is to be ensured with a restored configuration snapshot, that snapshot is checked whether or not + * it is of the same class as the cached one. The compatibility check is performed by simply ensuring that + * all nested serializers are compatible with the corresponding nested configuration snapshots. + */ +@Internal +public abstract class CompositeTypeSerializer extends TypeSerializer { + + private static final long serialVersionUID = -1460007327926564185L; + + private final CompositeTypeSerializerConfigSnapshot cachedConfigSnapshot; + + private final TypeSerializer[] nestedSerializers; + + /** + * Super constructor for composite type serializers. + * + * @param cachedConfigSnapshot the {@link CompositeTypeSerializerConfigSnapshot} to be cached. + * @param nestedSerializers the nested serializers captured by the cached config snapshot, in the exact same order. + */ + public CompositeTypeSerializer( + CompositeTypeSerializerConfigSnapshot cachedConfigSnapshot, + TypeSerializer... nestedSerializers) { + + this.cachedConfigSnapshot = Preconditions.checkNotNull(cachedConfigSnapshot); + this.nestedSerializers = Preconditions.checkNotNull(nestedSerializers); + + Preconditions.checkArgument( + cachedConfigSnapshot.getNumNestedSerializers() == nestedSerializers.length, + "The cached composite type serializer configuration snapshot captures a different number of" + + " nested serializers than the length of the provided list of nested serializers."); + } + + @Override + public final TypeSerializerConfigSnapshot snapshotConfiguration() { + return cachedConfigSnapshot; + } + + @Override + protected CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + if (isComparableSnapshot(configSnapshot)) { + @SuppressWarnings("unchecked") + CompositeTypeSerializerConfigSnapshot snapshot = (CompositeTypeSerializerConfigSnapshot) configSnapshot; + + int i = 0; + for (TypeSerializer nestedSerializer : nestedSerializers) { + CompatibilityResult compatibilityResult = CompatibilityUtil.resolveCompatibilityResult( + snapshot.getNestedSerializerConfigSnapshot(i), + nestedSerializer); + + if (compatibilityResult.isRequiresMigration()) { + return CompatibilityResult.requiresMigration(); + } + + i++; + } + + return CompatibilityResult.compatible(); + } else { + return CompatibilityResult.requiresMigration(); + } + } + + /** + * Subclasses can override this if the serializer recognizes configuration snapshot + * classes beyond the cached one. For example, composite type serializers which in previous versions + * return a different configuration snapshot class than the one currently used could override this + * method for backwards compatibility. + * + * @param configSnapshot the config snapshot to compare with. + */ + protected boolean isComparableSnapshot(TypeSerializerConfigSnapshot configSnapshot) { + return configSnapshot.getClass().equals(cachedConfigSnapshot.getClass()); + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java index 287137c6860a8..6e29d8348b092 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeSerializerConfigSnapshot.java @@ -40,7 +40,7 @@ @Internal public abstract class CompositeTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { - private List, TypeSerializerConfigSnapshot>> nestedSerializersAndConfigs; + private List> nestedSerializerConfigs; /** This empty nullary constructor is required for deserializing the configuration. */ public CompositeTypeSerializerConfigSnapshot() {} @@ -48,35 +48,56 @@ public CompositeTypeSerializerConfigSnapshot() {} public CompositeTypeSerializerConfigSnapshot(TypeSerializer... nestedSerializers) { Preconditions.checkNotNull(nestedSerializers); - this.nestedSerializersAndConfigs = new ArrayList<>(nestedSerializers.length); + this.nestedSerializerConfigs = new ArrayList<>(nestedSerializers.length); for (TypeSerializer nestedSerializer : nestedSerializers) { - TypeSerializerConfigSnapshot configSnapshot = nestedSerializer.snapshotConfiguration(); - this.nestedSerializersAndConfigs.add( - new Tuple2, TypeSerializerConfigSnapshot>( - nestedSerializer.duplicate(), - Preconditions.checkNotNull(configSnapshot))); + this.nestedSerializerConfigs.add( + Preconditions.checkNotNull( + nestedSerializer.snapshotConfiguration(), + "Configuration snapshots of nested serializers cannot be null.")); } } @Override public void write(DataOutputView out) throws IOException { super.write(out); - TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience(out, nestedSerializersAndConfigs); + TypeSerializerConfigSnapshotSerializationUtil.writeSerializerConfigSnapshots(out, nestedSerializerConfigs); } @Override public void read(DataInputView in) throws IOException { super.read(in); - this.nestedSerializersAndConfigs = - TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, getUserCodeClassLoader()); + + if (!containsSerializers()) { + this.nestedSerializerConfigs = TypeSerializerConfigSnapshotSerializationUtil + .readSerializerConfigSnapshots(in, getUserCodeClassLoader()); + } else { + // backwards compatible path + List, TypeSerializerConfigSnapshot>> nestedSerializersAndConfigs = + TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, getUserCodeClassLoader()); + + this.nestedSerializerConfigs = new ArrayList<>(nestedSerializersAndConfigs.size()); + for (Tuple2, TypeSerializerConfigSnapshot> entry : nestedSerializersAndConfigs) { + this.nestedSerializerConfigs.add(new BackwardsCompatibleConfigSnapshot<>(entry.f1, entry.f0)); + } + } + } + + /** + * Return whether or not this composite type serializer config snapshot still contains + * serializers. Subclasses should uptick their version, and use that to compare against the read version + * of the config snapshot to determine this. By default, it is assumed that all composite + * type serializer config snapshots do not contain serializers. + */ + protected boolean containsSerializers() { + return false; } - public List, TypeSerializerConfigSnapshot>> getNestedSerializersAndConfigs() { - return nestedSerializersAndConfigs; + public int getNumNestedSerializers() { + return nestedSerializerConfigs.size(); } - public Tuple2, TypeSerializerConfigSnapshot> getSingleNestedSerializerAndConfig() { - return nestedSerializersAndConfigs.get(0); + public TypeSerializerConfigSnapshot getNestedSerializerConfigSnapshot(int index) { + return nestedSerializerConfigs.get(index); } @Override @@ -90,11 +111,32 @@ public boolean equals(Object obj) { } return (obj.getClass().equals(getClass())) - && nestedSerializersAndConfigs.equals(((CompositeTypeSerializerConfigSnapshot) obj).getNestedSerializersAndConfigs()); + && nestedSerializerConfigs.equals(((CompositeTypeSerializerConfigSnapshot) obj).nestedSerializerConfigs); } @Override public int hashCode() { - return nestedSerializersAndConfigs.hashCode(); + return nestedSerializerConfigs.hashCode(); } + + @Override + public final TypeSerializer restoreSerializer() { + TypeSerializer[] restoredNestedSerializers = new TypeSerializer[nestedSerializerConfigs.size()]; + + int i = 0; + for (TypeSerializerConfigSnapshot config : nestedSerializerConfigs) { + restoredNestedSerializers[i] = config.restoreSerializer(); + i++; + } + + return restoreSerializer(restoredNestedSerializers); + } + + /** + * Restore the composite type serializer with the restored nested serializers. + * + * @param restoredNestedSerializers the restored nested serializers. + * @return the restored composite type serializer. + */ + protected abstract TypeSerializer restoreSerializer(TypeSerializer... restoredNestedSerializers); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java index d1a3e9508ac60..2d8802a85d1a7 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/CollectionSerializerConfigSnapshot.java @@ -28,12 +28,17 @@ * Configuration snapshot of a serializer for collection types. * * @param Type of the element. + * + * @deprecated this configuration snapshot is not capable of being a factory for all serializers that + * previously write this as their configuration snapshot, and therefore deprecated. It is no + * longer written by any of Flink's serializers, but is still here for backwards compatibility. */ @Internal +@Deprecated public final class CollectionSerializerConfigSnapshot, T> extends CompositeTypeSerializerConfigSnapshot { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public CollectionSerializerConfigSnapshot() {} @@ -42,8 +47,24 @@ public CollectionSerializerConfigSnapshot(TypeSerializer elementSerializer) { super(elementSerializer); } + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + // versions that still used this config snapshot always still wrote the serializers + return true; + } + @Override public int getVersion() { return VERSION; } + + @Override + protected TypeSerializer restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + throw new UnsupportedOperationException(); + } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java index 8e0969f350929..964377be8e0fa 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializer.java @@ -23,10 +23,10 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; + import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -38,7 +38,7 @@ * @param The component type. */ @Internal -public final class GenericArraySerializer extends TypeSerializer { +public final class GenericArraySerializer extends CompositeTypeSerializer { private static final long serialVersionUID = 1L; @@ -50,8 +50,13 @@ public final class GenericArraySerializer extends TypeSerializer { public GenericArraySerializer(Class componentClass, TypeSerializer componentSerializer) { - this.componentClass = checkNotNull(componentClass); - this.componentSerializer = checkNotNull(componentSerializer); + + super( + new GenericArraySerializerConfigSnapshot<>(checkNotNull(componentClass), checkNotNull(componentSerializer)), + componentSerializer); + + this.componentClass = componentClass; + this.componentSerializer = componentSerializer; } @Override @@ -195,27 +200,16 @@ public String toString() { // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - @Override - public GenericArraySerializerConfigSnapshot snapshotConfiguration() { - return new GenericArraySerializerConfigSnapshot<>(componentClass, componentSerializer); - } - + @SuppressWarnings("unchecked") @Override public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { if (configSnapshot instanceof GenericArraySerializerConfigSnapshot) { - final GenericArraySerializerConfigSnapshot config = (GenericArraySerializerConfigSnapshot) configSnapshot; - - if (componentClass.equals(config.getComponentClass())) { - Tuple2, TypeSerializerConfigSnapshot> previousComponentSerializerAndConfig = - config.getSingleNestedSerializerAndConfig(); + CompatibilityResult result = super.ensureCompatibility(configSnapshot); - CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousComponentSerializerAndConfig.f1, - componentSerializer); + final GenericArraySerializerConfigSnapshot config = (GenericArraySerializerConfigSnapshot) configSnapshot; - if (!compatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } + if (!result.isRequiresMigration() && componentClass.equals(config.getComponentClass())) { + return CompatibilityResult.compatible(); } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java index 5b207ddbb2586..d76750dc89273 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/GenericArraySerializerConfigSnapshot.java @@ -38,7 +38,7 @@ @Internal public final class GenericArraySerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { - private static final int VERSION = 1; + private static final int VERSION = 2; private Class componentClass; @@ -79,6 +79,11 @@ public int getVersion() { return VERSION; } + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + public Class getComponentClass() { return componentClass; } @@ -94,4 +99,17 @@ public boolean equals(Object obj) { public int hashCode() { return super.hashCode() * 31 + componentClass.hashCode(); } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new GenericArraySerializer<>( + componentClass, + (TypeSerializer) restoredNestedSerializers[0]); + } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java index 1ae0250ce2802..e7ca7c0b41c83 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializer.java @@ -19,11 +19,9 @@ package org.apache.flink.api.common.typeutils.base; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -43,7 +41,7 @@ * @param The type of element in the list. */ @Internal -public final class ListSerializer extends TypeSerializer> { +public final class ListSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 1119562170939152304L; @@ -56,7 +54,12 @@ public final class ListSerializer extends TypeSerializer> { * @param elementSerializer The serializer for the elements of the list */ public ListSerializer(TypeSerializer elementSerializer) { - this.elementSerializer = checkNotNull(elementSerializer); + + super( + new ListSerializerConfigSnapshot<>(checkNotNull(elementSerializer)), + elementSerializer); + + this.elementSerializer = elementSerializer; } // ------------------------------------------------------------------------ @@ -172,30 +175,11 @@ public int hashCode() { return elementSerializer.hashCode(); } - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // -------------------------------------------------------------------------------------------- - - @Override - public CollectionSerializerConfigSnapshot, T> snapshotConfiguration() { - return new CollectionSerializerConfigSnapshot<>(elementSerializer); - } - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof CollectionSerializerConfigSnapshot) { - Tuple2, TypeSerializerConfigSnapshot> previousElemSerializerAndConfig = - ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); - - CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousElemSerializerAndConfig.f1, - elementSerializer); - - if (!compatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); + protected boolean isComparableSnapshot(TypeSerializerConfigSnapshot configSnapshot) { + // previous versions of the ListSerializer still wrote the deprecated + // CollectionSerializerConfigSnapshot as the configuration snapshot + return configSnapshot instanceof ListSerializerConfigSnapshot + || configSnapshot instanceof CollectionSerializerConfigSnapshot; } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..0557762a17041 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/ListSerializerConfigSnapshot.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.typeutils.base; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; + +import java.util.List; + +/** + * A {@link TypeSerializerConfigSnapshot} for the {@link ListSerializer}. + */ +public class ListSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { + + private final int VERSION = 1; + + /** This empty nullary constructor is required for deserializing the configuration. */ + public ListSerializerConfigSnapshot() {} + + public ListSerializerConfigSnapshot(TypeSerializer elementSerializer) { + super(elementSerializer); + } + + @Override + public int getVersion() { + return VERSION; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new ListSerializer<>((TypeSerializer) restoredNestedSerializers[0]); + } +} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java index e010a905281ae..15ae10b0e7301 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializer.java @@ -19,17 +19,13 @@ package org.apache.flink.api.common.typeutils.base; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.util.Preconditions; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.HashMap; @@ -45,7 +41,7 @@ * @param The type of the values in the map. */ @Internal -public final class MapSerializer extends TypeSerializer> { +public final class MapSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = -6885593032367050078L; @@ -62,8 +58,16 @@ public final class MapSerializer extends TypeSerializer> { * @param valueSerializer The serializer for the values in the map */ public MapSerializer(TypeSerializer keySerializer, TypeSerializer valueSerializer) { - this.keySerializer = Preconditions.checkNotNull(keySerializer, "The key serializer cannot be null"); - this.valueSerializer = Preconditions.checkNotNull(valueSerializer, "The value serializer cannot be null."); + + super( + new MapSerializerConfigSnapshot<>( + Preconditions.checkNotNull(keySerializer, "The key serializer cannot be null."), + Preconditions.checkNotNull(valueSerializer, "The value serializer cannot be null.")), + keySerializer, + valueSerializer); + + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; } // ------------------------------------------------------------------------ @@ -199,35 +203,4 @@ public boolean canEqual(Object obj) { public int hashCode() { return keySerializer.hashCode() * 31 + valueSerializer.hashCode(); } - - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // -------------------------------------------------------------------------------------------- - - @Override - public MapSerializerConfigSnapshot snapshotConfiguration() { - return new MapSerializerConfigSnapshot<>(keySerializer, valueSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof MapSerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> previousKvSerializersAndConfigs = - ((MapSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - CompatibilityResult keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(0).f1, - keySerializer); - - CompatibilityResult valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(1).f1, - valueSerializer); - - if (!keyCompatResult.isRequiresMigration() && !valueCompatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java index e601d4d52cca3..05fc93bf372b6 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/MapSerializerConfigSnapshot.java @@ -31,7 +31,7 @@ @Internal public final class MapSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public MapSerializerConfigSnapshot() {} @@ -40,8 +40,26 @@ public MapSerializerConfigSnapshot(TypeSerializer keySerializer, TypeSerializ super(keySerializer, valueSerializer); } + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + @Override public int getVersion() { return VERSION; } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new MapSerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1]); + } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java index 4456b4de83286..acd9403c63c8f 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializer.java @@ -19,17 +19,14 @@ package org.apache.flink.api.java.typeutils.runtime; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.types.Either; +import org.apache.flink.util.Preconditions; import java.io.IOException; -import java.util.List; import static org.apache.flink.types.Either.Left; import static org.apache.flink.types.Either.Right; @@ -41,7 +38,7 @@ * @param the Right value type */ @Internal -public class EitherSerializer extends TypeSerializer> { +public class EitherSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 1L; @@ -50,6 +47,14 @@ public class EitherSerializer extends TypeSerializer> { private final TypeSerializer rightSerializer; public EitherSerializer(TypeSerializer leftSerializer, TypeSerializer rightSerializer) { + + super( + new EitherSerializerConfigSnapshot<>( + Preconditions.checkNotNull(leftSerializer), + Preconditions.checkNotNull(rightSerializer)), + leftSerializer, + rightSerializer); + this.leftSerializer = leftSerializer; this.rightSerializer = rightSerializer; } @@ -188,35 +193,4 @@ public boolean canEqual(Object obj) { public int hashCode() { return 17 * leftSerializer.hashCode() + rightSerializer.hashCode(); } - - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // -------------------------------------------------------------------------------------------- - - @Override - public EitherSerializerConfigSnapshot snapshotConfiguration() { - return new EitherSerializerConfigSnapshot<>(leftSerializer, rightSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof EitherSerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> previousLeftRightSerializersAndConfigs = - ((EitherSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - CompatibilityResult leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerializersAndConfigs.get(0).f1, - leftSerializer); - - CompatibilityResult rightCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerializersAndConfigs.get(1).f1, - rightSerializer); - - if (!leftCompatResult.isRequiresMigration() && !rightCompatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java index 628fff4110778..f228e87309848 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/EitherSerializerConfigSnapshot.java @@ -30,7 +30,7 @@ @Internal public final class EitherSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public EitherSerializerConfigSnapshot() {} @@ -43,4 +43,22 @@ public EitherSerializerConfigSnapshot(TypeSerializer leftSerializer, TypeSeri public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new EitherSerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1]); + } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java index a7c3b18a4ae4f..8bc5352cc4ae5 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/RowSerializer.java @@ -18,12 +18,9 @@ package org.apache.flink.api.java.typeutils.runtime; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.types.Row; @@ -31,7 +28,6 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.util.Arrays; -import java.util.List; import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoAndCopyNullMask; import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.readIntoNullMask; @@ -42,7 +38,7 @@ * Serializer for {@link Row}. */ @Internal -public final class RowSerializer extends TypeSerializer { +public final class RowSerializer extends CompositeTypeSerializer { private static final long serialVersionUID = 1L; @@ -54,7 +50,10 @@ public final class RowSerializer extends TypeSerializer { @SuppressWarnings("unchecked") public RowSerializer(TypeSerializer[] fieldSerializers) { - this.fieldSerializers = (TypeSerializer[]) checkNotNull(fieldSerializers); + + super(new RowSerializerConfigSnapshot(checkNotNull(fieldSerializers)), fieldSerializers); + + this.fieldSerializers = (TypeSerializer[]) fieldSerializers; this.arity = fieldSerializers.length; this.nullMask = new boolean[fieldSerializers.length]; } @@ -260,46 +259,14 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - @Override - public RowSerializerConfigSnapshot snapshotConfiguration() { - return new RowSerializerConfigSnapshot(fieldSerializers); - } - - @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof RowSerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> previousFieldSerializersAndConfigs = - ((RowSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - if (previousFieldSerializersAndConfigs.size() == fieldSerializers.length) { - - CompatibilityResult compatResult; - int i = 0; - for (Tuple2, TypeSerializerConfigSnapshot> f : previousFieldSerializersAndConfigs) { - compatResult = CompatibilityUtil.resolveCompatibilityResult(f.f1, fieldSerializers[i]); - - if (compatResult.isRequiresMigration()) { - return CompatibilityResult.requiresMigration(); - } - - i++; - } - - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); - } - public static final class RowSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public RowSerializerConfigSnapshot() {} - public RowSerializerConfigSnapshot(TypeSerializer[] fieldSerializers) { + public RowSerializerConfigSnapshot(TypeSerializer[] fieldSerializers) { super(fieldSerializers); } @@ -307,5 +274,20 @@ public RowSerializerConfigSnapshot(TypeSerializer[] fieldSerializers) { public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @Override + protected TypeSerializer restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new RowSerializer(restoredNestedSerializers); + } } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializer.java index 911c96f393b68..46daae7ff8e94 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializer.java @@ -21,7 +21,9 @@ import java.io.IOException; import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.CompatibilityResult; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -33,7 +35,10 @@ public class TupleSerializer extends TupleSerializerBase { private static final long serialVersionUID = 1L; public TupleSerializer(Class tupleClass, TypeSerializer[] fieldSerializers) { - super(tupleClass, fieldSerializers); + super( + tupleClass, + new TupleSerializerConfigSnapshot<>(tupleClass, fieldSerializers), + fieldSerializers); } @Override @@ -147,6 +152,20 @@ public T deserialize(T reuse, DataInputView source) throws IOException { } return reuse; } + + @SuppressWarnings("unchecked") + @Override + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + CompatibilityResult result = super.ensureCompatibility(configSnapshot); + + final TupleSerializerConfigSnapshot config = (TupleSerializerConfigSnapshot) configSnapshot; + + if (!result.isRequiresMigration() && !tupleClass.equals(config.getTupleClass())) { + return CompatibilityResult.requiresMigration(); + } + + return result; + } private T instantiateRaw() { try { @@ -156,9 +175,4 @@ private T instantiateRaw() { throw new RuntimeException("Cannot instantiate tuple.", e); } } - - @Override - protected TupleSerializerBase createSerializerInstance(Class tupleClass, TypeSerializer[] fieldSerializers) { - return new TupleSerializer<>(tupleClass, fieldSerializers); - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java index fdcd828ca3643..bcf0063148600 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerBase.java @@ -20,23 +20,20 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import java.io.IOException; import java.util.Arrays; -import java.util.List; import java.util.Objects; import static org.apache.flink.util.Preconditions.checkNotNull; @Internal -public abstract class TupleSerializerBase extends TypeSerializer { +public abstract class TupleSerializerBase extends CompositeTypeSerializer { private static final long serialVersionUID = 1L; @@ -49,9 +46,15 @@ public abstract class TupleSerializerBase extends TypeSerializer { private int length = -2; @SuppressWarnings("unchecked") - public TupleSerializerBase(Class tupleClass, TypeSerializer[] fieldSerializers) { + public TupleSerializerBase( + Class tupleClass, + CompositeTypeSerializerConfigSnapshot serializerConfigSnapshot, + TypeSerializer[] fieldSerializers) { + + super(serializerConfigSnapshot, fieldSerializers); + this.tupleClass = checkNotNull(tupleClass); - this.fieldSerializers = (TypeSerializer[]) checkNotNull(fieldSerializers); + this.fieldSerializers = (TypeSerializer[]) fieldSerializers; this.arity = fieldSerializers.length; } @@ -122,49 +125,6 @@ public boolean canEqual(Object obj) { return obj instanceof TupleSerializerBase; } - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // -------------------------------------------------------------------------------------------- - - @Override - public TupleSerializerConfigSnapshot snapshotConfiguration() { - return new TupleSerializerConfigSnapshot<>(tupleClass, fieldSerializers); - } - - @SuppressWarnings("unchecked") - @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof TupleSerializerConfigSnapshot) { - final TupleSerializerConfigSnapshot config = (TupleSerializerConfigSnapshot) configSnapshot; - - if (tupleClass.equals(config.getTupleClass())) { - List, TypeSerializerConfigSnapshot>> previousFieldSerializersAndConfigs = - ((TupleSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - if (previousFieldSerializersAndConfigs.size() == fieldSerializers.length) { - - CompatibilityResult compatResult; - int i = 0; - for (Tuple2, TypeSerializerConfigSnapshot> f : previousFieldSerializersAndConfigs) { - compatResult = CompatibilityUtil.resolveCompatibilityResult(f.f1, fieldSerializers[i]); - - if (compatResult.isRequiresMigration()) { - return CompatibilityResult.requiresMigration(); - } - - i++; - } - - return CompatibilityResult.compatible(); - } - } - } - - return CompatibilityResult.requiresMigration(); - } - - protected abstract TupleSerializerBase createSerializerInstance(Class tupleClass, TypeSerializer[] fieldSerializers); - @VisibleForTesting public TypeSerializer[] getFieldSerializers() { return fieldSerializers; diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java index a1180aeba5325..1ff2824d821ac 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.util.InstantiationUtil; @@ -29,12 +30,12 @@ import java.io.IOException; /** - * Snapshot of a tuple serializer's configuration. + * Snapshot of a {@link TupleSerializer}'s configuration. */ @Internal -public final class TupleSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { +public final class TupleSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { - private static final int VERSION = 1; + private static final int VERSION = 2; private Class tupleClass; @@ -72,6 +73,21 @@ public int getVersion() { return VERSION; } + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @Override + protected TypeSerializer restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new TupleSerializer<>(tupleClass, restoredNestedSerializers); + } + public Class getTupleClass() { return tupleClass; } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java index 1fe4a865fae7c..59cc2d5b01b2c 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/NFA.java @@ -19,8 +19,7 @@ package org.apache.flink.cep.nfa; import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; @@ -863,7 +862,7 @@ public Queue getComputationStates() { @Deprecated public static final class NFASerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty constructor is required for deserializing the configuration. */ public NFASerializerConfigSnapshot() {} @@ -879,13 +878,31 @@ public NFASerializerConfigSnapshot( public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer... restoredNestedSerializers) { + return new NFASerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer>) restoredNestedSerializers[1]); + } } /** * Only for backward compatibility with <=1.5. */ @Deprecated - public static class NFASerializer extends TypeSerializer> { + public static class NFASerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 2098282423980597010L; @@ -903,6 +920,14 @@ public NFASerializer(TypeSerializer typeSerializer) { NFASerializer( TypeSerializer typeSerializer, TypeSerializer> sharedBufferSerializer) { + + super( + new NFASerializerConfigSnapshot<>( + Preconditions.checkNotNull(typeSerializer), + Preconditions.checkNotNull(sharedBufferSerializer)), + typeSerializer, + sharedBufferSerializer); + this.eventSerializer = typeSerializer; this.sharedBufferSerializer = sharedBufferSerializer; } @@ -983,33 +1008,5 @@ public boolean canEqual(Object obj) { public int hashCode() { return 37 * sharedBufferSerializer.hashCode() + eventSerializer.hashCode(); } - - @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new NFASerializerConfigSnapshot<>(eventSerializer, sharedBufferSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof NFASerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> serializersAndConfigs = - ((NFASerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - CompatibilityResult eventCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializersAndConfigs.get(0).f1, - eventSerializer); - - CompatibilityResult> sharedBufCompatResult = - CompatibilityUtil.resolveCompatibilityResult( - serializersAndConfigs.get(1).f1, - sharedBufferSerializer); - - if (!sharedBufCompatResult.isRequiresMigration() && !eventCompatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); - } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java index 6fc50bc86bd32..864681523cdf7 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/SharedBuffer.java @@ -18,8 +18,7 @@ package org.apache.flink.cep.nfa; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; @@ -32,6 +31,7 @@ import org.apache.flink.cep.nfa.sharedbuffer.SharedBufferNode; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; import java.io.IOException; import java.util.ArrayList; @@ -150,7 +150,7 @@ public static ValueTimeWrapper deserialize( public static final class SharedBufferSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty constructor is required for deserializing the configuration. */ public SharedBufferSerializerConfigSnapshot() { @@ -168,12 +168,32 @@ public SharedBufferSerializerConfigSnapshot( public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new SharedBufferSerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1], + (TypeSerializer) restoredNestedSerializers[2]); + } } /** * A {@link TypeSerializer} for the {@link SharedBuffer}. */ - public static class SharedBufferSerializer extends TypeSerializer> { + public static class SharedBufferSerializer + extends CompositeTypeSerializer> { private static final long serialVersionUID = -3254176794680331560L; @@ -192,6 +212,15 @@ public SharedBufferSerializer( final TypeSerializer valueSerializer, final TypeSerializer versionSerializer) { + super( + new SharedBufferSerializerConfigSnapshot<>( + Preconditions.checkNotNull(keySerializer), + Preconditions.checkNotNull(valueSerializer), + Preconditions.checkNotNull(versionSerializer)), + keySerializer, + valueSerializer, + versionSerializer); + this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; this.versionSerializer = versionSerializer; @@ -339,42 +368,5 @@ public boolean canEqual(Object obj) { public int hashCode() { return 37 * keySerializer.hashCode() + valueSerializer.hashCode(); } - - @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new SharedBufferSerializerConfigSnapshot<>( - keySerializer, - valueSerializer, - versionSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof SharedBufferSerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> serializerConfigSnapshots = - ((SharedBufferSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - CompatibilityResult keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializerConfigSnapshots.get(0).f1, - keySerializer); - - CompatibilityResult valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializerConfigSnapshots.get(1).f1, - valueSerializer); - - CompatibilityResult versionCompatResult = CompatibilityUtil.resolveCompatibilityResult( - serializerConfigSnapshots.get(2).f1, - versionSerializer); - - if (!keyCompatResult.isRequiresMigration() && !valueCompatResult.isRequiresMigration() && - !versionCompatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } else { - return CompatibilityResult.requiresMigration(); - } - } - - return CompatibilityResult.requiresMigration(); - } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java index 40b015639731e..d8a8ba721e3ef 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/Lockable.java @@ -18,13 +18,12 @@ package org.apache.flink.cep.nfa.sharedbuffer; -import org.apache.flink.api.common.typeutils.CompatibilityResult; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.base.IntSerializer; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.util.Preconditions; import java.io.IOException; import java.util.Objects; @@ -92,11 +91,15 @@ public int hashCode() { } /** Serializer for {@link Lockable}. */ - public static class LockableTypeSerializer extends TypeSerializer> { + public static class LockableTypeSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 3298801058463337340L; private final TypeSerializer elementSerializer; LockableTypeSerializer(TypeSerializer elementSerializer) { + super( + new LockableSerializerConfigSnapshot<>(Preconditions.checkNotNull(elementSerializer)), + elementSerializer); + this.elementSerializer = elementSerializer; } @@ -179,31 +182,5 @@ public int hashCode() { public boolean canEqual(Object obj) { return obj.getClass().equals(LockableTypeSerializer.class); } - - @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new LockableSerializerConfigSnapshot<>(elementSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof LockableSerializerConfigSnapshot) { - @SuppressWarnings("unchecked") - LockableSerializerConfigSnapshot snapshot = (LockableSerializerConfigSnapshot) configSnapshot; - - Tuple2, TypeSerializerConfigSnapshot> nestedSerializerAndConfig = - snapshot.getSingleNestedSerializerAndConfig(); - - CompatibilityResult inputComaptibilityResult = elementSerializer - .internalEnsureCompatibility(nestedSerializerAndConfig.f1); - if (inputComaptibilityResult.isRequiresMigration()) { - return CompatibilityResult.requiresMigration(); - } else { - return CompatibilityResult.compatible(); - } - } else { - return CompatibilityResult.requiresMigration(); - } - } } } diff --git a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java index f3702ac0459a7..9942801dcc973 100644 --- a/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java +++ b/flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/LockableSerializerConfigSnapshot.java @@ -40,4 +40,9 @@ public int getVersion() { return VERSION; } + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new Lockable.LockableTypeSerializer<>((TypeSerializer) restoredNestedSerializers[0]); + } } diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java index 6230e811c4658..12e89b32b5755 100644 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/ListViewSerializerConfigSnapshot.java @@ -19,6 +19,7 @@ package org.apache.flink.table.dataview; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.base.ListSerializer; import org.apache.flink.table.api.dataview.ListView; @@ -40,4 +41,10 @@ public ListViewSerializerConfigSnapshot(ListSerializer listSerializer) { public int getVersion() { return VERSION; } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new ListViewSerializer<>((ListSerializer) restoredNestedSerializers[0]); + } } diff --git a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java index 36954b95b236a..315544500b109 100644 --- a/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java +++ b/flink-libraries/flink-table/src/main/java/org/apache/flink/table/dataview/MapViewSerializerConfigSnapshot.java @@ -19,6 +19,7 @@ package org.apache.flink.table.dataview; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.base.MapSerializer; import org.apache.flink.table.api.dataview.MapView; @@ -41,4 +42,10 @@ public MapViewSerializerConfigSnapshot(MapSerializer mapSerializer) { public int getVersion() { return VERSION; } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new MapViewSerializer<>((MapSerializer) restoredNestedSerializers[0]); + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala index b8f631fd067f1..81dac4912c9b5 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeutils._ import org.apache.flink.api.common.typeutils.base.{CollectionSerializerConfigSnapshot, ListSerializer} import org.apache.flink.core.memory.{DataInputView, DataOutputView} import org.apache.flink.table.api.dataview.ListView +import org.apache.flink.util.Preconditions /** * A serializer for [[ListView]]. The serializer relies on an element @@ -34,7 +35,9 @@ import org.apache.flink.table.api.dataview.ListView * @tparam T The type of element in the list. */ class ListViewSerializer[T](val listSerializer: ListSerializer[T]) - extends TypeSerializer[ListView[T]] { + extends CompositeTypeSerializer[ListView[T]]( + new ListViewSerializerConfigSnapshot[T](Preconditions.checkNotNull(listSerializer)), + listSerializer) { override def isImmutableType: Boolean = false @@ -75,40 +78,13 @@ class ListViewSerializer[T](val listSerializer: ListSerializer[T]) override def equals(obj: Any): Boolean = canEqual(this) && listSerializer.equals(obj.asInstanceOf[ListViewSerializer[_]].listSerializer) - override def snapshotConfiguration(): ListViewSerializerConfigSnapshot[T] = - new ListViewSerializerConfigSnapshot[T](listSerializer) - - override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[ListView[T]] = { - - configSnapshot match { - case snapshot: ListViewSerializerConfigSnapshot[T] => - checkCompatibility(snapshot) + override def isComparableSnapshot( + configSnapshot: TypeSerializerConfigSnapshot[_]): Boolean = { + configSnapshot.isInstanceOf[ListViewSerializerConfigSnapshot[T]] || // backwards compatibility path; // Flink versions older or equal to 1.5.x returns a // CollectionSerializerConfigSnapshot as the snapshot - case legacySnapshot: CollectionSerializerConfigSnapshot[java.util.List[T], T] => - checkCompatibility(legacySnapshot) - - case _ => CompatibilityResult.requiresMigration[ListView[T]] - } - } - - private def checkCompatibility( - configSnapshot: CompositeTypeSerializerConfigSnapshot[_] - ): CompatibilityResult[ListView[T]] = { - - val previousListSerializerAndConfig = configSnapshot.getSingleNestedSerializerAndConfig - - val compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousListSerializerAndConfig.f1, - listSerializer.getElementSerializer) - - if (!compatResult.isRequiresMigration) { - CompatibilityResult.compatible[ListView[T]] - } else { - CompatibilityResult.requiresMigration[ListView[T]] - } + configSnapshot.isInstanceOf[CollectionSerializerConfigSnapshot[_, T]] } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala index d624a40944c85..044f2fd47b291 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala @@ -22,6 +22,7 @@ import org.apache.flink.api.common.typeutils._ import org.apache.flink.api.common.typeutils.base.{MapSerializer, MapSerializerConfigSnapshot} import org.apache.flink.core.memory.{DataInputView, DataOutputView} import org.apache.flink.table.api.dataview.MapView +import org.apache.flink.util.Preconditions /** * A serializer for [[MapView]]. The serializer relies on a key serializer and a value @@ -36,7 +37,9 @@ import org.apache.flink.table.api.dataview.MapView * @tparam V The type of the values in the map. */ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) - extends TypeSerializer[MapView[K, V]] { + extends CompositeTypeSerializer[MapView[K, V]]( + new MapViewSerializerConfigSnapshot[K, V](Preconditions.checkNotNull(mapSerializer)), + mapSerializer) { override def isImmutableType: Boolean = false @@ -77,45 +80,13 @@ class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V]) override def equals(obj: Any): Boolean = canEqual(this) && mapSerializer.equals(obj.asInstanceOf[MapViewSerializer[_, _]].mapSerializer) - override def snapshotConfiguration(): MapViewSerializerConfigSnapshot[K, V] = - new MapViewSerializerConfigSnapshot[K, V](mapSerializer) - - // copy and modified from MapSerializer.ensureCompatibility - override def ensureCompatibility(configSnapshot: TypeSerializerConfigSnapshot[_]) - : CompatibilityResult[MapView[K, V]] = { - - configSnapshot match { - case snapshot: MapViewSerializerConfigSnapshot[K, V] => - checkCompatibility(snapshot) + override def isComparableSnapshot( + configSnapshot: TypeSerializerConfigSnapshot[_]): Boolean = { + configSnapshot.isInstanceOf[MapViewSerializerConfigSnapshot[K, V]] || // backwards compatibility path; // Flink versions older or equal to 1.5.x returns a // MapSerializerConfigSnapshot as the snapshot - case legacySnapshot: MapSerializerConfigSnapshot[K, V] => - checkCompatibility(legacySnapshot) - - case _ => CompatibilityResult.requiresMigration[MapView[K, V]] - } - } - - private def checkCompatibility( - configSnapshot: CompositeTypeSerializerConfigSnapshot[_] - ): CompatibilityResult[MapView[K, V]] = { - - val previousKvSerializersAndConfigs = configSnapshot.getNestedSerializersAndConfigs - - val keyCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(0).f1, - mapSerializer.getKeySerializer) - - val valueCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousKvSerializersAndConfigs.get(1).f1, - mapSerializer.getValueSerializer) - - if (!keyCompatResult.isRequiresMigration && !valueCompatResult.isRequiresMigration) { - CompatibilityResult.compatible[MapView[K, V]] - } else { - CompatibilityResult.requiresMigration[MapView[K, V]] - } + configSnapshot.isInstanceOf[MapSerializerConfigSnapshot[K, V]] } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala index a7f1f4748bb8d..c3dd73fa071b4 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/types/CRowSerializer.scala @@ -90,7 +90,7 @@ class CRowSerializer(val rowSerializer: TypeSerializer[Row]) extends TypeSeriali configSnapshot match { case crowSerializerConfigSnapshot: CRowSerializer.CRowSerializerConfigSnapshot => val compatResult = CompatibilityUtil.resolveCompatibilityResult( - crowSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f1, + crowSerializerConfigSnapshot.getNestedSerializerConfigSnapshot(0), rowSerializer) if (compatResult.isRequiresMigration) { @@ -114,10 +114,26 @@ object CRowSerializer { def this() = this(null) override def getVersion: Int = CRowSerializerConfigSnapshot.VERSION + + override def restoreSerializer( + restoredNestedSerializers: TypeSerializer[_]* + ): TypeSerializer[CRow] = { + + new CRowSerializer( + restoredNestedSerializers(0).asInstanceOf[TypeSerializer[Row]]) + } + + override def containsSerializers(): Boolean = { + getReadVersion < 2 + } + + override def getCompatibleVersions: Array[Int] = { + Array(CRowSerializerConfigSnapshot.VERSION, 1) + } } object CRowSerializerConfigSnapshot { - val VERSION = 1 + val VERSION = 2 } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java index d30864c6c5d89..c60a3b402b49a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializer.java @@ -17,26 +17,31 @@ */ package org.apache.flink.runtime.state; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.base.CollectionSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import java.io.IOException; import java.util.ArrayList; +import static org.apache.flink.util.Preconditions.checkNotNull; + @SuppressWarnings("ForLoopReplaceableByForEach") -final public class ArrayListSerializer extends TypeSerializer> { +final public class ArrayListSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 1119562170939152304L; private final TypeSerializer elementSerializer; public ArrayListSerializer(TypeSerializer elementSerializer) { + + super( + new ArrayListSerializerConfigSnapshot<>(checkNotNull(elementSerializer)), + elementSerializer); + this.elementSerializer = elementSerializer; } @@ -138,30 +143,9 @@ public int hashCode() { return elementSerializer.hashCode(); } - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // -------------------------------------------------------------------------------------------- - - @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new CollectionSerializerConfigSnapshot<>(elementSerializer); - } - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof CollectionSerializerConfigSnapshot) { - Tuple2, TypeSerializerConfigSnapshot> previousElemSerializerAndConfig = - ((CollectionSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); - - CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousElemSerializerAndConfig.f1, - elementSerializer); - - if (!compatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); + protected boolean isComparableSnapshot(TypeSerializerConfigSnapshot configSnapshot) { + return configSnapshot instanceof ArrayListSerializerConfigSnapshot + || configSnapshot instanceof CollectionSerializerConfigSnapshot; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerConfigSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..b17205193c5d2 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/ArrayListSerializerConfigSnapshot.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.state; + +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; + +import java.util.ArrayList; + +/** + * A {@link TypeSerializerConfigSnapshot} for the {@link ArrayListSerializer}. + */ +public class ArrayListSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { + + private final int VERSION = 1; + + public ArrayListSerializerConfigSnapshot(TypeSerializer elementSerializer) { + super(elementSerializer); + } + + @Override + public int getVersion() { + return VERSION; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new ArrayListSerializer<>((TypeSerializer) restoredNestedSerializers[0]); + } +} diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java index 9566a663169b0..02f83c5669163 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaEitherSerializerConfigSnapshot.java @@ -43,4 +43,12 @@ public ScalaEitherSerializerConfigSnapshot(TypeSerializer leftSerializer, Typ public int getVersion() { return VERSION; } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer restoreSerializer(TypeSerializer... restoredNestedSerializers) { + return new EitherSerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1]); + } } diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java index 215bd447c05d4..85d2496fc15f4 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaOptionSerializerConfigSnapshot.java @@ -33,7 +33,7 @@ */ public final class ScalaOptionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public ScalaOptionSerializerConfigSnapshot() {} @@ -46,4 +46,20 @@ public ScalaOptionSerializerConfigSnapshot(TypeSerializer elementSerializer) public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new OptionSerializer<>((TypeSerializer) restoredNestedSerializers[0]); + } } diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java index 72baca44193bb..c51790c94ffc3 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/ScalaTrySerializerConfigSnapshot.java @@ -33,7 +33,7 @@ */ public class ScalaTrySerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public ScalaTrySerializerConfigSnapshot() {} @@ -49,4 +49,22 @@ public ScalaTrySerializerConfigSnapshot( public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new TrySerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1]); + } } diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java index b7f9ca6ec1afb..54ea72b6f8312 100644 --- a/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java +++ b/flink-scala/src/main/java/org/apache/flink/api/scala/typeutils/TraversableSerializerConfigSnapshot.java @@ -47,4 +47,10 @@ public TraversableSerializerConfigSnapshot(TypeSerializer elementSerializer) public int getVersion() { return VERSION; } + + @Override + protected TypeSerializer restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + // TODO to be implemented in follow-up commits + return null; + } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala index 74f5f5bef8f8f..5338afbee716d 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala @@ -149,15 +149,6 @@ private[flink] trait TypeInformationGen[C <: Context] { override def createInstance(fields: Array[AnyRef]): T = { instance.splice } - - override def createSerializerInstance( - tupleClass: Class[T], - fieldSerializers: Array[TypeSerializer[_]]) = { - this.getClass - .getConstructors()(0) - .newInstance(tupleClass, fieldSerializers) - .asInstanceOf[CaseClassSerializer[T]] - } } } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/package.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/package.scala index 4fd559ad6edfd..0832b69625c62 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/package.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/package.scala @@ -118,11 +118,5 @@ package object scala { override def createInstance(fields: Array[AnyRef]) = { (fields(0).asInstanceOf[T1], fields(1).asInstanceOf[T2]) } - - override def createSerializerInstance( - tupleClass: Class[(T1, T2)], - fieldSerializers: Array[TypeSerializer[_]]) = { - new Tuple2CaseClassSerializer[T1, T2](tupleClass, fieldSerializers) - } } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassSerializer.scala index c0599138be547..892cb1dab7314 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassSerializer.scala @@ -32,7 +32,9 @@ import org.apache.flink.types.NullFieldException abstract class CaseClassSerializer[T <: Product]( clazz: Class[T], scalaFieldSerializers: Array[TypeSerializer[_]]) - extends TupleSerializerBase[T](clazz, scalaFieldSerializers) + // TODO configuration snapshot of case class serializers is to + // TODO be implemented in follow-up commits + extends TupleSerializerBase[T](clazz, null, scalaFieldSerializers) with Cloneable { @transient var fields : Array[AnyRef] = _ @@ -81,15 +83,6 @@ abstract class CaseClassSerializer[T <: Product]( createInstance(fields) } - override def createSerializerInstance( - tupleClass: Class[T], - fieldSerializers: Array[TypeSerializer[_]]): TupleSerializerBase[T] = { - this.getClass - .getConstructors()(0) - .newInstance(tupleClass, fieldSerializers) - .asInstanceOf[CaseClassSerializer[T]] - } - def copy(from: T, reuse: T): T = { copy(from) } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala index 5049da45eb92e..4e61730b526f2 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/EitherSerializer.scala @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal import org.apache.flink.api.common.typeutils._ import org.apache.flink.api.java.typeutils.runtime.EitherSerializerConfigSnapshot import org.apache.flink.core.memory.{DataInputView, DataOutputView} +import org.apache.flink.util.Preconditions /** * Serializer for [[Either]]. @@ -30,7 +31,12 @@ import org.apache.flink.core.memory.{DataInputView, DataOutputView} class EitherSerializer[A, B, T <: Either[A, B]]( val leftSerializer: TypeSerializer[A], val rightSerializer: TypeSerializer[B]) - extends TypeSerializer[T] { + extends CompositeTypeSerializer[T]( + new ScalaEitherSerializerConfigSnapshot[T, A, B]( + Preconditions.checkNotNull(leftSerializer), + Preconditions.checkNotNull(rightSerializer)), + leftSerializer, + rightSerializer) { override def duplicate: EitherSerializer[A,B,T] = this @@ -111,47 +117,13 @@ class EitherSerializer[A, B, T <: Either[A, B]]( // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - override def snapshotConfiguration(): ScalaEitherSerializerConfigSnapshot[T, A, B] = { - new ScalaEitherSerializerConfigSnapshot[T, A, B](leftSerializer, rightSerializer) - } - - override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[T] = { - - configSnapshot match { - case eitherSerializerConfig: ScalaEitherSerializerConfigSnapshot[T, A, B] => - checkCompatibility(eitherSerializerConfig) + override def isComparableSnapshot( + configSnapshot: TypeSerializerConfigSnapshot[_]): Boolean = { + configSnapshot.isInstanceOf[ScalaEitherSerializerConfigSnapshot[T, A, B]] || // backwards compatibility path; // Flink versions older or equal to 1.5.x uses a // EitherSerializerConfigSnapshot as the snapshot - case legacyConfig: EitherSerializerConfigSnapshot[A, B] => - checkCompatibility(legacyConfig) - - case _ => CompatibilityResult.requiresMigration() - } - } - - private def checkCompatibility( - configSnapshot: CompositeTypeSerializerConfigSnapshot[_] - ): CompatibilityResult[T] = { - - val previousLeftRightSerWithConfigs = - configSnapshot.getNestedSerializersAndConfigs - - val leftCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerWithConfigs.get(0).f1, - leftSerializer) - - val rightCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousLeftRightSerWithConfigs.get(1).f1, - rightSerializer) - - if (leftCompatResult.isRequiresMigration - || rightCompatResult.isRequiresMigration) { - CompatibilityResult.requiresMigration() - } else { - CompatibilityResult.compatible() - } + configSnapshot.isInstanceOf[EitherSerializerConfigSnapshot[A, B]] } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala index c7ac4e95081d6..707fde8d95036 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/OptionSerializer.scala @@ -20,6 +20,7 @@ package org.apache.flink.api.scala.typeutils import org.apache.flink.annotation.Internal import org.apache.flink.api.common.typeutils._ import org.apache.flink.core.memory.{DataInputView, DataOutputView} +import org.apache.flink.util.Preconditions /** * Serializer for [[Option]]. @@ -27,7 +28,9 @@ import org.apache.flink.core.memory.{DataInputView, DataOutputView} @Internal @SerialVersionUID(-8635243274072627338L) class OptionSerializer[A](val elemSerializer: TypeSerializer[A]) - extends TypeSerializer[Option[A]] { + extends CompositeTypeSerializer[Option[A]]( + new ScalaOptionSerializerConfigSnapshot[A](Preconditions.checkNotNull(elemSerializer)), + elemSerializer) { override def duplicate: OptionSerializer[A] = { val duplicatedElemSerializer = elemSerializer.duplicate() @@ -97,41 +100,11 @@ class OptionSerializer[A](val elemSerializer: TypeSerializer[A]) elemSerializer.hashCode() } - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // -------------------------------------------------------------------------------------------- + override def isComparableSnapshot( + configSnapshot: TypeSerializerConfigSnapshot[_]): Boolean = { - override def snapshotConfiguration(): ScalaOptionSerializerConfigSnapshot[A] = { - new ScalaOptionSerializerConfigSnapshot[A](elemSerializer) - } - - override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Option[A]] = { - - configSnapshot match { - case optionSerializerConfigSnapshot - : ScalaOptionSerializerConfigSnapshot[A] => - ensureCompatibility(optionSerializerConfigSnapshot) - case legacyOptionSerializerConfigSnapshot - : OptionSerializer.OptionSerializerConfigSnapshot[A] => - ensureCompatibility(legacyOptionSerializerConfigSnapshot) - case _ => CompatibilityResult.requiresMigration() - } - } - - private def ensureCompatibility( - compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot[Option[A]]) - : CompatibilityResult[Option[A]] = { - - val compatResult = CompatibilityUtil.resolveCompatibilityResult( - compositeConfigSnapshot.getSingleNestedSerializerAndConfig.f1, - elemSerializer) - - if (compatResult.isRequiresMigration) { - CompatibilityResult.requiresMigration() - } else { - CompatibilityResult.compatible() - } + configSnapshot.isInstanceOf[ScalaOptionSerializerConfigSnapshot[A]] || + configSnapshot.isInstanceOf[OptionSerializer.OptionSerializerConfigSnapshot[A]] } } @@ -145,10 +118,26 @@ object OptionSerializer { extends CompositeTypeSerializerConfigSnapshot[Option[A]] { override def getVersion: Int = OptionSerializerConfigSnapshot.VERSION + + override def restoreSerializer( + restoredNestedSerializers: TypeSerializer[_]* + ): TypeSerializer[Option[A]] = { + + new OptionSerializer[A]( + restoredNestedSerializers(0).asInstanceOf[TypeSerializer[A]]) + } + + override def containsSerializers(): Boolean = { + getReadVersion < 2 + } + + override def getCompatibleVersions: Array[Int] = { + Array(getVersion, 1) + } } object OptionSerializerConfigSnapshot { - val VERSION = 1 + val VERSION = 2 } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala index 574dc0474c4aa..962ba2d245fa5 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala @@ -164,7 +164,7 @@ abstract class TraversableSerializer[T <: TraversableOnce[E], E]( : TraversableSerializerConfigSnapshot[T, E] => val elemCompatRes = CompatibilityUtil.resolveCompatibilityResult( - traversableSerializerConfigSnapshot.getSingleNestedSerializerAndConfig.f1, + traversableSerializerConfigSnapshot.getNestedSerializerConfigSnapshot(0), elementSerializer) if (elemCompatRes.isRequiresMigration) { diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala index 8f4c2a2c3447d..ca07f911fb756 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TrySerializer.scala @@ -18,10 +18,9 @@ package org.apache.flink.api.scala.typeutils import org.apache.flink.annotation.Internal -import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.api.common.typeutils._ -import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer import org.apache.flink.core.memory.{DataInputView, DataOutputView} +import org.apache.flink.util.Preconditions import scala.util.{Failure, Success, Try} @@ -32,13 +31,16 @@ import scala.util.{Failure, Success, Try} @SerialVersionUID(-3052182891252564491L) class TrySerializer[A]( private val elemSerializer: TypeSerializer[A], - private val executionConfig: ExecutionConfig) - extends TypeSerializer[Try[A]] { + private val throwableSerializer: TypeSerializer[Throwable]) + extends CompositeTypeSerializer[Try[A]]( + new ScalaTrySerializerConfigSnapshot[A]( + Preconditions.checkNotNull(elemSerializer), + Preconditions.checkNotNull(throwableSerializer)), + elemSerializer, + throwableSerializer) { override def duplicate: TrySerializer[A] = this - val throwableSerializer = new KryoSerializer[Throwable](classOf[Throwable], executionConfig) - override def createInstance: Try[A] = { Failure(new RuntimeException("Empty Failure")) } @@ -97,51 +99,20 @@ class TrySerializer[A]( } override def hashCode(): Int = { - 31 * elemSerializer.hashCode() + executionConfig.hashCode() + 31 * elemSerializer.hashCode() + throwableSerializer.hashCode() } // -------------------------------------------------------------------------------------------- // Serializer configuration snapshotting & compatibility // -------------------------------------------------------------------------------------------- - override def snapshotConfiguration(): ScalaTrySerializerConfigSnapshot[A] = { - new ScalaTrySerializerConfigSnapshot[A](elemSerializer, throwableSerializer) - } - - override def ensureCompatibility( - configSnapshot: TypeSerializerConfigSnapshot[_]): CompatibilityResult[Try[A]] = { - - configSnapshot match { - case trySerializerConfigSnapshot - : ScalaTrySerializerConfigSnapshot[A] => - ensureCompatibility(trySerializerConfigSnapshot) - case legacyTrySerializerConfigSnapshot - : TrySerializer.TrySerializerConfigSnapshot[A] => - ensureCompatibility(legacyTrySerializerConfigSnapshot) - case _ => CompatibilityResult.requiresMigration() - } - } - - private def ensureCompatibility( - compositeConfigSnapshot: CompositeTypeSerializerConfigSnapshot[Try[A]]) - : CompatibilityResult[Try[A]] = { - - val previousSerializersAndConfigs = - compositeConfigSnapshot.getNestedSerializersAndConfigs - - val elemCompatRes = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(0).f1, - elemSerializer) - - val throwableCompatRes = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(1).f1, - throwableSerializer) - - if (elemCompatRes.isRequiresMigration || throwableCompatRes.isRequiresMigration) { - CompatibilityResult.requiresMigration() - } else { - CompatibilityResult.compatible() - } + override def isComparableSnapshot( + configSnapshot: TypeSerializerConfigSnapshot[_]): Boolean = { + configSnapshot.isInstanceOf[ScalaTrySerializerConfigSnapshot[A]] || + // backwards compatibility path; + // Flink versions older or equal to 1.5.x returns a + // TrySerializer.TrySerializerConfigSnapshot as the snapshot + configSnapshot.isInstanceOf[TrySerializer.TrySerializerConfigSnapshot[A]] } } @@ -155,10 +126,23 @@ object TrySerializer { extends CompositeTypeSerializerConfigSnapshot[Try[A]]() { override def getVersion: Int = TrySerializerConfigSnapshot.VERSION + + override def restoreSerializer( + restoredNestedSerializers: TypeSerializer[_]* + ): TypeSerializer[Try[A]] = { + + new TrySerializer[A]( + restoredNestedSerializers(0).asInstanceOf[TypeSerializer[A]], + restoredNestedSerializers(1).asInstanceOf[TypeSerializer[Throwable]]) + } + + override def containsSerializers(): Boolean = { + getReadVersion < 2 + } } object TrySerializerConfigSnapshot { - val VERSION = 1 + val VERSION = 2 } } diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TryTypeInfo.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TryTypeInfo.scala index b09c353dfab63..89c95cbd6bbc6 100644 --- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TryTypeInfo.scala +++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TryTypeInfo.scala @@ -17,13 +17,13 @@ */ package org.apache.flink.api.scala.typeutils -import org.apache.flink.annotation.{PublicEvolving, Public} +import org.apache.flink.annotation.{Public, PublicEvolving} import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.common.typeutils.TypeSerializer +import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer import scala.collection.JavaConverters._ - import scala.util.Try /** @@ -52,10 +52,15 @@ class TryTypeInfo[A, T <: Try[A]](val elemTypeInfo: TypeInformation[A]) def createSerializer(executionConfig: ExecutionConfig): TypeSerializer[T] = { if (elemTypeInfo == null) { // this happens when the type of a DataSet is None, i.e. DataSet[Failure] - new TrySerializer(new NothingSerializer, executionConfig).asInstanceOf[TypeSerializer[T]] + new TrySerializer( + new NothingSerializer, + new KryoSerializer[Throwable](classOf[Throwable], executionConfig) + ).asInstanceOf[TypeSerializer[T]] } else { - new TrySerializer(elemTypeInfo.createSerializer(executionConfig), executionConfig) - .asInstanceOf[TypeSerializer[T]] + new TrySerializer( + elemTypeInfo.createSerializer(executionConfig), + new KryoSerializer[Throwable](classOf[Throwable], executionConfig) + ).asInstanceOf[TypeSerializer[T]] } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java index 01122c9e215f0..ba50b69f6cd6e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/CoGroupedStreams.java @@ -24,14 +24,12 @@ import org.apache.flink.api.common.functions.CoGroupFunction; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.operators.translation.WrappingFunction; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -41,6 +39,7 @@ import org.apache.flink.streaming.api.windowing.triggers.Trigger; import org.apache.flink.streaming.api.windowing.windows.Window; import org.apache.flink.util.Collector; +import org.apache.flink.util.Preconditions; import java.io.IOException; import java.util.ArrayList; @@ -437,13 +436,21 @@ public boolean canEqual(Object obj) { } } - private static class UnionSerializer extends TypeSerializer> { + private static class UnionSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 1L; private final TypeSerializer oneSerializer; private final TypeSerializer twoSerializer; public UnionSerializer(TypeSerializer oneSerializer, TypeSerializer twoSerializer) { + + super( + new UnionSerializerConfigSnapshot<>( + Preconditions.checkNotNull(oneSerializer), + Preconditions.checkNotNull(twoSerializer)), + oneSerializer, + twoSerializer); + this.oneSerializer = oneSerializer; this.twoSerializer = twoSerializer; } @@ -549,33 +556,6 @@ public boolean equals(Object obj) { public boolean canEqual(Object obj) { return obj instanceof UnionSerializer; } - - @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new UnionSerializerConfigSnapshot<>(oneSerializer, twoSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof UnionSerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> previousSerializersAndConfigs = - ((UnionSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - CompatibilityResult oneSerializerCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(0).f1, - oneSerializer); - - CompatibilityResult twoSerializerCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(1).f1, - twoSerializer); - - if (!oneSerializerCompatResult.isRequiresMigration() && !twoSerializerCompatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); - } } /** @@ -584,7 +564,7 @@ public CompatibilityResult> ensureCompatibility(TypeSerializ public static class UnionSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public UnionSerializerConfigSnapshot() {} @@ -597,6 +577,24 @@ public UnionSerializerConfigSnapshot(TypeSerializer oneSerializer, TypeSeria public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new UnionSerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1]); + } } // ------------------------------------------------------------------------ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java index 3e41ae23aa621..68e86f2eaaa8c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunction.java @@ -24,12 +24,10 @@ import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeHint; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.runtime.state.CheckpointListener; @@ -587,7 +585,7 @@ public String toString() { */ @VisibleForTesting @Internal - public static final class StateSerializer extends TypeSerializer> { + public static final class StateSerializer extends CompositeTypeSerializer> { private static final long serialVersionUID = 1L; @@ -597,8 +595,16 @@ public static final class StateSerializer extends TypeSerializer transactionSerializer, TypeSerializer contextSerializer) { - this.transactionSerializer = checkNotNull(transactionSerializer); - this.contextSerializer = checkNotNull(contextSerializer); + + super( + new StateSerializerConfigSnapshot<>( + checkNotNull(transactionSerializer), + checkNotNull(contextSerializer)), + transactionSerializer, + contextSerializer); + + this.transactionSerializer = transactionSerializer; + this.contextSerializer = contextSerializer; } @Override @@ -759,34 +765,6 @@ public int hashCode() { result = 31 * result + contextSerializer.hashCode(); return result; } - - @Override - public TypeSerializerConfigSnapshot> snapshotConfiguration() { - return new StateSerializerConfigSnapshot<>(transactionSerializer, contextSerializer); - } - - @Override - public CompatibilityResult> ensureCompatibility( - TypeSerializerConfigSnapshot configSnapshot) { - if (configSnapshot instanceof StateSerializerConfigSnapshot) { - List, TypeSerializerConfigSnapshot>> previousSerializersAndConfigs = - ((StateSerializerConfigSnapshot) configSnapshot).getNestedSerializersAndConfigs(); - - CompatibilityResult txnCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(0).f1, - transactionSerializer); - - CompatibilityResult contextCompatResult = CompatibilityUtil.resolveCompatibilityResult( - previousSerializersAndConfigs.get(1).f1, - contextSerializer); - - if (!txnCompatResult.isRequiresMigration() && !contextCompatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } - } - - return CompatibilityResult.requiresMigration(); - } } /** @@ -798,7 +776,7 @@ public CompatibilityResult> ensureCompatibility( public static final class StateSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot> { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public StateSerializerConfigSnapshot() {} @@ -813,5 +791,23 @@ public StateSerializerConfigSnapshot( public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer> restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new StateSerializer<>( + (TypeSerializer) restoredNestedSerializers[0], + (TypeSerializer) restoredNestedSerializers[1]); + } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java index 93366d47d41b2..8b1ef0b1811b8 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamrecord/StreamElementSerializer.java @@ -19,22 +19,18 @@ package org.apache.flink.streaming.runtime.streamrecord; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.CompatibilityUtil; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializer; import org.apache.flink.api.common.typeutils.CompositeTypeSerializerConfigSnapshot; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; +import org.apache.flink.util.Preconditions; import java.io.IOException; -import static java.util.Objects.requireNonNull; - /** * Serializer for {@link StreamRecord}, {@link Watermark}, {@link LatencyMarker}, and * {@link StreamStatus}. @@ -45,7 +41,7 @@ * @param The type of value in the StreamRecord */ @Internal -public final class StreamElementSerializer extends TypeSerializer { +public final class StreamElementSerializer extends CompositeTypeSerializer { private static final long serialVersionUID = 1L; @@ -59,10 +55,13 @@ public final class StreamElementSerializer extends TypeSerializer typeSerializer; public StreamElementSerializer(TypeSerializer serializer) { + + super(new StreamElementSerializerConfigSnapshot<>(Preconditions.checkNotNull(serializer)), serializer); + if (serializer instanceof StreamElementSerializer) { throw new RuntimeException("StreamRecordSerializer given to StreamRecordSerializer as value TypeSerializer: " + serializer); } - this.typeSerializer = requireNonNull(serializer); + this.typeSerializer = serializer; } public TypeSerializer getContainedTypeSerializer() { @@ -270,48 +269,12 @@ public int hashCode() { return typeSerializer.hashCode(); } - // -------------------------------------------------------------------------------------------- - // Serializer configuration snapshotting & compatibility - // - // This serializer may be used by Flink internal operators that need to checkpoint - // buffered records. Therefore, it may be part of managed state and need to implement - // the configuration snapshot and compatibility methods. - // -------------------------------------------------------------------------------------------- - - @Override - public StreamElementSerializerConfigSnapshot snapshotConfiguration() { - return new StreamElementSerializerConfigSnapshot<>(typeSerializer); - } - - @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - Tuple2, TypeSerializerConfigSnapshot> previousTypeSerializerAndConfig; - - // we are compatible for data written by ourselves or the legacy MultiplexingStreamRecordSerializer - if (configSnapshot instanceof StreamElementSerializerConfigSnapshot) { - previousTypeSerializerAndConfig = - ((StreamElementSerializerConfigSnapshot) configSnapshot).getSingleNestedSerializerAndConfig(); - } else { - return CompatibilityResult.requiresMigration(); - } - - CompatibilityResult compatResult = CompatibilityUtil.resolveCompatibilityResult( - previousTypeSerializerAndConfig.f1, - typeSerializer); - - if (!compatResult.isRequiresMigration()) { - return CompatibilityResult.compatible(); - } else { - return CompatibilityResult.requiresMigration(); - } - } - /** * Configuration snapshot specific to the {@link StreamElementSerializer}. */ public static final class StreamElementSerializerConfigSnapshot extends CompositeTypeSerializerConfigSnapshot { - private static final int VERSION = 1; + private static final int VERSION = 2; /** This empty nullary constructor is required for deserializing the configuration. */ public StreamElementSerializerConfigSnapshot() {} @@ -324,5 +287,21 @@ public StreamElementSerializerConfigSnapshot(TypeSerializer typeSerializer) { public int getVersion() { return VERSION; } + + @Override + public int[] getCompatibleVersions() { + return new int[]{VERSION, 1}; + } + + @Override + protected boolean containsSerializers() { + return getReadVersion() < 2; + } + + @SuppressWarnings("unchecked") + @Override + protected TypeSerializer restoreSerializer(TypeSerializer[] restoredNestedSerializers) { + return new StreamElementSerializer<>((TypeSerializer) restoredNestedSerializers[0]); + } } }