From 96260813832841c71e2dc5095bd10c878fd6040c Mon Sep 17 00:00:00 2001 From: ConradJam Date: Mon, 6 May 2024 17:34:34 +0800 Subject: [PATCH] [AMORO-2226]: Rename Flink Model Arctic to Amoro --- .../apache/amoro/flink/FlinkSchemaUtil.java | 4 +- .../amoro/flink/catalog/MixedCatalog.java | 6 +- ...n.java => AmoroRowDataLookupFunction.java} | 8 +- .../flink/lookup/BasicLookupFunction.java | 8 +- .../{ArcticSource.java => AmoroSource.java} | 73 ++++++----- .../amoro/flink/read/FlinkSplitPlanner.java | 26 ++-- .../flink/read/MixedIncrementalLoader.java | 10 +- .../flink/read/PartitionAndNodeGroup.java | 8 +- .../hybrid/assigner/ShuffleSplitAssigner.java | 45 ++++--- .../flink/read/hybrid/assigner/Split.java | 12 +- .../read/hybrid/assigner/SplitAssigner.java | 12 +- .../hybrid/assigner/StaticSplitAssigner.java | 26 ++-- ...ator.java => AbstractAmoroEnumerator.java} | 18 +-- ...Offset.java => AmoroEnumeratorOffset.java} | 14 +-- ...a => AmoroEnumeratorOffsetSerializer.java} | 19 ++- ...umState.java => AmoroSourceEnumState.java} | 20 +-- ...va => AmoroSourceEnumStateSerializer.java} | 40 +++--- ...erator.java => AmoroSourceEnumerator.java} | 50 ++++---- .../ContinuousEnumerationResult.java | 25 ++-- .../enumerator/ContinuousSplitPlanner.java | 4 +- .../ContinuousSplitPlannerImpl.java | 26 ++-- .../InitializationFinishedEvent.java | 4 +- .../MergeOnReadIncrementalPlanner.java | 14 +-- .../enumerator/MergeOnReadPlannerImpl.java | 18 +-- ....java => StaticAmoroSourceEnumerator.java} | 33 +++-- ...rdEmitter.java => AmoroRecordEmitter.java} | 12 +- ...Offset.java => AmoroRecordWithOffset.java} | 2 +- ...urceReader.java => AmoroSourceReader.java} | 26 ++-- .../read/hybrid/reader/ArrayBatchRecords.java | 12 +- .../reader/ArrayPoolDataIteratorBatcher.java | 6 +- .../hybrid/reader/DataIteratorBatcher.java | 2 +- .../reader/DataIteratorReaderFunction.java | 7 +- .../read/hybrid/reader/HybridSplitReader.java | 16 +-- .../read/hybrid/reader/ReaderFunction.java | 8 +- .../hybrid/reader/ReaderStartedEvent.java | 2 +- .../hybrid/reader/RowDataReaderFunction.java | 24 ++-- .../{ArcticSplit.java => AmoroSplit.java} | 6 +- ...ializer.java => AmoroSplitSerializer.java} | 10 +- ...icSplitState.java => AmoroSplitState.java} | 8 +- .../read/hybrid/split/ChangelogSplit.java | 4 +- .../read/hybrid/split/MergeOnReadSplit.java | 4 +- .../read/hybrid/split/SnapshotSplit.java | 4 +- .../read/hybrid/split/TemporalJoinSplits.java | 4 +- ...ScanContext.java => AmoroScanContext.java} | 10 +- ...aReader.java => FlinkAmoroDataReader.java} | 6 +- ...ader.java => FlinkAmoroMORDataReader.java} | 4 +- .../read/source/MergeOnReadDataIterator.java | 4 +- ...DynamicSink.java => AmoroDynamicSink.java} | 17 ++- ...micSource.java => AmoroDynamicSource.java} | 24 ++-- ...icFileSource.java => AmoroFileSource.java} | 18 +-- ...TableLoader.java => AmoroTableLoader.java} | 22 ++-- .../flink/table/DynamicTableFactory.java | 26 ++-- .../apache/amoro/flink/table/FlinkSource.java | 20 +-- .../{ArcticUtils.java => AmoroUtils.java} | 14 +-- ...icFileWriter.java => AmoroFileWriter.java} | 22 ++-- ...cticLogWriter.java => AmoroLogWriter.java} | 4 +- ...ava => AmoroRowDataTaskWriterFactory.java} | 6 +- .../{ArcticWriter.java => AmoroWriter.java} | 10 +- .../write/AutomaticDoubleWriteStatus.java | 10 +- .../amoro/flink/write/AutomaticLogWriter.java | 30 ++--- .../apache/amoro/flink/write/FlinkSink.java | 54 ++++----- .../write/hidden/AbstractHiddenLogWriter.java | 4 +- ...titioner.java => AmoroLogPartitioner.java} | 4 +- .../hidden/kafka/HiddenKafkaFactory.java | 4 +- .../hidden/kafka/HiddenKafkaProducer.java | 10 +- .../amoro/flink/FlinkTableTestBase.java | 12 +- .../org/apache/amoro/flink/FlinkTestBase.java | 6 +- ...ArcticSource.java => TestAmoroSource.java} | 114 +++++++++--------- .../amoro/flink/read/TestFlinkSource.java | 14 +-- .../flink/read/TestFlinkSplitPlanner.java | 8 +- .../assigner/TestShuffleSplitAssigner.java | 24 ++-- .../assigner/TestSplitAssignerAwaiting.java | 14 +-- .../assigner/TestStaticSplitAssigner.java | 10 +- ...> TestAmoroSourceEnumStateSerializer.java} | 22 ++-- ...or.java => TestAmoroSourceEnumerator.java} | 42 +++---- .../TestTemporalJoinSplitsThreadSafe.java | 26 ++-- .../reader/MixedIncrementalLoaderTest.java | 8 +- .../reader/TestRowDataReaderFunction.java | 6 +- ...zer.java => TestAmoroSplitSerializer.java} | 18 +-- .../apache/amoro/flink/table/TestJoin.java | 6 +- .../amoro/flink/table/TestTableRefresh.java | 4 +- .../amoro/flink/table/TestWatermark.java | 10 +- ...tArctic.java => MockEnvironmentAmoro.java} | 4 +- .../flink/util/MockEnvironmentBuilder.java | 2 +- ...ITCase.java => AmoroFileWriterITCase.java} | 22 ++-- .../flink/write/FlinkTaskWriterBaseTest.java | 4 +- ...itter.java => TestAmoroFileCommitter.java} | 18 +-- ...leWriter.java => TestAmoroFileWriter.java} | 30 ++--- .../write/TestAutomaticDoubleWriteStatus.java | 6 +- .../flink/write/TestAutomaticLogWriter.java | 18 +-- .../amoro/flink/write/TestFlinkSink.java | 10 +- 91 files changed, 724 insertions(+), 737 deletions(-) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/{ArcticRowDataLookupFunction.java => AmoroRowDataLookupFunction.java} (93%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/{ArcticSource.java => AmoroSource.java} (56%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{AbstractArcticEnumerator.java => AbstractAmoroEnumerator.java} (92%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{ArcticEnumeratorOffset.java => AmoroEnumeratorOffset.java} (83%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{ArcticEnumeratorOffsetSerializer.java => AmoroEnumeratorOffsetSerializer.java} (77%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{ArcticSourceEnumState.java => AmoroSourceEnumState.java} (77%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{ArcticSourceEnumStateSerializer.java => AmoroSourceEnumStateSerializer.java} (77%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{ArcticSourceEnumerator.java => AmoroSourceEnumerator.java} (84%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/{StaticArcticSourceEnumerator.java => StaticAmoroSourceEnumerator.java} (72%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/{ArcticRecordEmitter.java => AmoroRecordEmitter.java} (85%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/{ArcticRecordWithOffset.java => AmoroRecordWithOffset.java} (97%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/{ArcticSourceReader.java => AmoroSourceReader.java} (87%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/{ArcticSplit.java => AmoroSplit.java} (93%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/{ArcticSplitSerializer.java => AmoroSplitSerializer.java} (90%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/{ArcticSplitState.java => AmoroSplitState.java} (94%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/{ArcticScanContext.java => AmoroScanContext.java} (97%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/{FlinkArcticDataReader.java => FlinkAmoroDataReader.java} (96%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/{FlinkArcticMORDataReader.java => FlinkAmoroMORDataReader.java} (96%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/{ArcticDynamicSink.java => AmoroDynamicSink.java} (87%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/{ArcticDynamicSource.java => AmoroDynamicSource.java} (96%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/{ArcticFileSource.java => AmoroFileSource.java} (95%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/{ArcticTableLoader.java => AmoroTableLoader.java} (91%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/{ArcticUtils.java => AmoroUtils.java} (96%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/{ArcticFileWriter.java => AmoroFileWriter.java} (91%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/{ArcticLogWriter.java => AmoroLogWriter.java} (90%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/{ArcticRowDataTaskWriterFactory.java => AmoroRowDataTaskWriterFactory.java} (91%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/{ArcticWriter.java => AmoroWriter.java} (96%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/{ArcticLogPartitioner.java => AmoroLogPartitioner.java} (94%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/{TestArcticSource.java => TestAmoroSource.java} (92%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/{TestArcticSourceEnumStateSerializer.java => TestAmoroSourceEnumStateSerializer.java} (83%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/{TestArcticSourceEnumerator.java => TestAmoroSourceEnumerator.java} (88%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/{TestArcticSplitSerializer.java => TestAmoroSplitSerializer.java} (83%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/{MockEnvironmentArctic.java => MockEnvironmentAmoro.java} (96%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/{ArcticFileWriterITCase.java => AmoroFileWriterITCase.java} (93%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/{TestArcticFileCommitter.java => TestAmoroFileCommitter.java} (90%) rename amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/{TestArcticFileWriter.java => TestAmoroFileWriter.java} (92%) diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/FlinkSchemaUtil.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/FlinkSchemaUtil.java index 6451a28651..bec331c108 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/FlinkSchemaUtil.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/FlinkSchemaUtil.java @@ -26,7 +26,7 @@ import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_EXPR; import static org.apache.flink.table.descriptors.Schema.SCHEMA_PROCTIME; -import org.apache.amoro.flink.table.ArcticDynamicSource; +import org.apache.amoro.flink.table.AmoroDynamicSource; import org.apache.amoro.flink.table.FlinkSource; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.PrimaryKeySpec; @@ -116,7 +116,7 @@ public static TableSchema toSchema( } /** - * Add watermark info to help {@link FlinkSource} and {@link ArcticDynamicSource} distinguish the + * Add watermark info to help {@link FlinkSource} and {@link AmoroDynamicSource} distinguish the * watermark field. For now, it only be used in the case of Arctic as dim-table. */ public static TableSchema getPhysicalSchemaForDimTable(TableSchema tableSchema) { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/catalog/MixedCatalog.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/catalog/MixedCatalog.java index ed7a0d3bac..bcc0f4619d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/catalog/MixedCatalog.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/catalog/MixedCatalog.java @@ -29,7 +29,7 @@ import org.apache.amoro.flink.catalog.factories.CatalogFactoryOptions; import org.apache.amoro.flink.table.DynamicTableFactory; import org.apache.amoro.flink.table.descriptors.ArcticValidator; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.scan.CombinedScanTask; import org.apache.amoro.scan.KeyedTableScanTask; @@ -207,7 +207,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) List partitionKeys = toPartitionKeys(table.spec(), table.schema()); return CatalogTable.of( - toSchema(arcticSchema, ArcticUtils.getPrimaryKeys(table), arcticProperties).toSchema(), + toSchema(arcticSchema, AmoroUtils.getPrimaryKeys(table), arcticProperties).toSchema(), null, partitionKeys, arcticProperties); @@ -740,7 +740,7 @@ private CatalogTable toCatalogTable(MixedTable table, TableIdentifier tableIdent List partitionKeys = toPartitionKeys(table.spec(), table.schema()); return new CatalogTableImpl( - toSchema(arcticSchema, ArcticUtils.getPrimaryKeys(table), arcticProperties), + toSchema(arcticSchema, AmoroUtils.getPrimaryKeys(table), arcticProperties), partitionKeys, arcticProperties, null); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/ArcticRowDataLookupFunction.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/AmoroRowDataLookupFunction.java similarity index 93% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/ArcticRowDataLookupFunction.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/AmoroRowDataLookupFunction.java index 13fa44e624..a2a69e505e 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/ArcticRowDataLookupFunction.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/AmoroRowDataLookupFunction.java @@ -19,7 +19,7 @@ package org.apache.amoro.flink.lookup; import org.apache.amoro.flink.read.hybrid.reader.DataIteratorReaderFunction; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.hive.io.reader.AbstractAdaptHiveKeyedDataReader; import org.apache.amoro.table.MixedTable; import org.apache.flink.configuration.Configuration; @@ -35,17 +35,17 @@ import java.util.function.Predicate; /** A lookup function for {@link RowData} type. */ -public class ArcticRowDataLookupFunction extends LookupFunction { +public class AmoroRowDataLookupFunction extends LookupFunction { private static final long serialVersionUID = -7694050999266540499L; private final BasicLookupFunction basicLookupFunction; - public ArcticRowDataLookupFunction( + public AmoroRowDataLookupFunction( TableFactory tableFactory, MixedTable mixedTable, List joinKeys, Schema projectSchema, List filters, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, Configuration config, Predicate predicate, AbstractAdaptHiveKeyedDataReader flinkArcticMORDataReader, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/BasicLookupFunction.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/BasicLookupFunction.java index 2df14e3a8a..7601f069d2 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/BasicLookupFunction.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/BasicLookupFunction.java @@ -19,13 +19,13 @@ package org.apache.amoro.flink.lookup; import static org.apache.amoro.flink.table.descriptors.ArcticValidator.LOOKUP_RELOADING_INTERVAL; -import static org.apache.amoro.flink.util.ArcticUtils.loadArcticTable; +import static org.apache.amoro.flink.util.AmoroUtils.loadArcticTable; import static org.apache.flink.util.Preconditions.checkArgument; import org.apache.amoro.flink.read.MixedIncrementalLoader; import org.apache.amoro.flink.read.hybrid.enumerator.MergeOnReadIncrementalPlanner; import org.apache.amoro.flink.read.hybrid.reader.DataIteratorReaderFunction; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.hive.io.reader.AbstractAdaptHiveKeyedDataReader; import org.apache.amoro.table.MixedTable; import org.apache.flink.configuration.Configuration; @@ -65,7 +65,7 @@ public class BasicLookupFunction implements Serializable { private final List joinKeys; private final Schema projectSchema; private final List filters; - private final ArcticTableLoader loader; + private final AmoroTableLoader loader; private long nextLoadTime = Long.MIN_VALUE; private final long reloadIntervalSeconds; private MixedIncrementalLoader incrementalLoader; @@ -85,7 +85,7 @@ public BasicLookupFunction( List joinKeys, Schema projectSchema, List filters, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, Configuration config, Predicate predicate, AbstractAdaptHiveKeyedDataReader flinkArcticMORDataReader, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/ArcticSource.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/AmoroSource.java similarity index 56% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/ArcticSource.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/AmoroSource.java index 2ba5acfae4..26cc8ae392 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/ArcticSource.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/AmoroSource.java @@ -21,16 +21,16 @@ import org.apache.amoro.flink.read.hybrid.assigner.ShuffleSplitAssigner; import org.apache.amoro.flink.read.hybrid.assigner.SplitAssigner; import org.apache.amoro.flink.read.hybrid.assigner.StaticSplitAssigner; -import org.apache.amoro.flink.read.hybrid.enumerator.ArcticSourceEnumState; -import org.apache.amoro.flink.read.hybrid.enumerator.ArcticSourceEnumStateSerializer; -import org.apache.amoro.flink.read.hybrid.enumerator.ArcticSourceEnumerator; -import org.apache.amoro.flink.read.hybrid.enumerator.StaticArcticSourceEnumerator; -import org.apache.amoro.flink.read.hybrid.reader.ArcticSourceReader; +import org.apache.amoro.flink.read.hybrid.enumerator.AmoroSourceEnumState; +import org.apache.amoro.flink.read.hybrid.enumerator.AmoroSourceEnumStateSerializer; +import org.apache.amoro.flink.read.hybrid.enumerator.AmoroSourceEnumerator; +import org.apache.amoro.flink.read.hybrid.enumerator.StaticAmoroSourceEnumerator; +import org.apache.amoro.flink.read.hybrid.reader.AmoroSourceReader; import org.apache.amoro.flink.read.hybrid.reader.ReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitSerializer; -import org.apache.amoro.flink.read.source.ArcticScanContext; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitSerializer; +import org.apache.amoro.flink.read.source.AmoroScanContext; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.api.connector.source.Source; @@ -42,29 +42,29 @@ import org.apache.flink.core.io.SimpleVersionedSerializer; /** - * Arctic Source based of FLIP-27. + * Amoro Source based of FLIP-27. * - *

If ArcticSource is used as a build table in lookup join, it will be implemented by temporal - * join. Two source should use processing time as watermark. ArcticSource will generate watermark - * after first splits planned by ArcticSourceEnumerator having been finished. + *

If AmoroSource is used as a build table in lookup join, it will be implemented by temporal + * join. Two source should use processing time as watermark. AmoroSource will generate watermark + * after first splits planned by AmoroSourceEnumerator having been finished. */ -public class ArcticSource - implements Source, ResultTypeQueryable { +public class AmoroSource + implements Source, ResultTypeQueryable { private static final long serialVersionUID = 1L; - private final ArcticScanContext scanContext; + private final AmoroScanContext scanContext; private final ReaderFunction readerFunction; private final TypeInformation typeInformation; - private final ArcticTableLoader loader; + private final AmoroTableLoader loader; private final String tableName; /** - * generate arctic watermark. This is only for lookup join arctic table, and arctic table is used - * as build table, i.e. right table. + * generate amoro watermark. This is only for lookup join amoro table, and amoro table is used as + * build table, i.e. right table. */ private final boolean dimTable; - public ArcticSource( - ArcticTableLoader loader, - ArcticScanContext scanContext, + public AmoroSource( + AmoroTableLoader loader, + AmoroScanContext scanContext, ReaderFunction readerFunction, TypeInformation typeInformation, String tableName, @@ -83,45 +83,44 @@ public Boundedness getBoundedness() { } @Override - public SourceReader createReader(SourceReaderContext readerContext) { - return new ArcticSourceReader<>( + public SourceReader createReader(SourceReaderContext readerContext) { + return new AmoroSourceReader<>( readerFunction, readerContext.getConfiguration(), readerContext, dimTable); } @Override - public SplitEnumerator createEnumerator( - SplitEnumeratorContext enumContext) { + public SplitEnumerator createEnumerator( + SplitEnumeratorContext enumContext) { return createEnumerator(enumContext, null); } - private SplitEnumerator createEnumerator( - SplitEnumeratorContext enumContext, ArcticSourceEnumState enumState) { + private SplitEnumerator createEnumerator( + SplitEnumeratorContext enumContext, AmoroSourceEnumState enumState) { SplitAssigner splitAssigner; if (scanContext.isStreaming()) { splitAssigner = new ShuffleSplitAssigner(enumContext, tableName, enumState); - return new ArcticSourceEnumerator( + return new AmoroSourceEnumerator( enumContext, splitAssigner, loader, scanContext, enumState, dimTable); } else { splitAssigner = new StaticSplitAssigner(enumState); - return new StaticArcticSourceEnumerator( - enumContext, splitAssigner, loader, scanContext, null); + return new StaticAmoroSourceEnumerator(enumContext, splitAssigner, loader, scanContext, null); } } @Override - public SplitEnumerator restoreEnumerator( - SplitEnumeratorContext enumContext, ArcticSourceEnumState checkpoint) { + public SplitEnumerator restoreEnumerator( + SplitEnumeratorContext enumContext, AmoroSourceEnumState checkpoint) { return createEnumerator(enumContext, checkpoint); } @Override - public SimpleVersionedSerializer getSplitSerializer() { - return new ArcticSplitSerializer(); + public SimpleVersionedSerializer getSplitSerializer() { + return new AmoroSplitSerializer(); } @Override - public SimpleVersionedSerializer getEnumeratorCheckpointSerializer() { - return new ArcticSourceEnumStateSerializer(); + public SimpleVersionedSerializer getEnumeratorCheckpointSerializer() { + return new AmoroSourceEnumStateSerializer(); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/FlinkSplitPlanner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/FlinkSplitPlanner.java index f50f4d8d7c..9c598cbcd4 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/FlinkSplitPlanner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/FlinkSplitPlanner.java @@ -19,7 +19,7 @@ package org.apache.amoro.flink.read; import org.apache.amoro.data.DataFileType; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.flink.read.hybrid.split.MergeOnReadSplit; import org.apache.amoro.flink.read.hybrid.split.SnapshotSplit; @@ -59,7 +59,7 @@ public class FlinkSplitPlanner { private FlinkSplitPlanner() {} - public static List planFullTable(KeyedTable keyedTable, AtomicInteger splitCount) { + public static List planFullTable(KeyedTable keyedTable, AtomicInteger splitCount) { CloseableIterable combinedScanTasks = keyedTable.newScan().planTasks(); BaseAndChangeTask baseAndChangeTask = BaseAndChangeTask.of(combinedScanTasks); return planFullTable(baseAndChangeTask, splitCount); @@ -72,10 +72,10 @@ public static List planFullTable(KeyedTable keyedTable, AtomicInteg * @param keyedTable The {@link KeyedTable} to scan. * @param filters Optional list of filters to apply to the scan. * @param splitCount The atomic integer to track the split count. - * @return The list of planned {@link ArcticSplit} included {@link SnapshotSplit}, {@link + * @return The list of planned {@link AmoroSplit} included {@link SnapshotSplit}, {@link * ChangelogSplit}. */ - public static List planFullTable( + public static List planFullTable( KeyedTable keyedTable, List filters, AtomicInteger splitCount) { KeyedTableScan keyedTableScan = keyedTable.newScan(); if (filters != null) { @@ -86,10 +86,10 @@ public static List planFullTable( return planFullTable(baseAndChangeTask, splitCount); } - private static List planFullTable( + private static List planFullTable( BaseAndChangeTask baseAndChangeTask, AtomicInteger splitCount) { Collection baseTasks = baseAndChangeTask.allBaseTasks(); - List allSplits = + List allSplits = baseTasks.stream() .map( arcticFileScanTask -> @@ -98,7 +98,7 @@ private static List planFullTable( .collect(Collectors.toList()); Collection changeTasks = baseAndChangeTask.transactionTasks(); - List changeSplits = planChangeTable(changeTasks, splitCount); + List changeSplits = planChangeTable(changeTasks, splitCount); allSplits.addAll(changeSplits); return allSplits; @@ -111,16 +111,16 @@ private static List planFullTable( * @param keyedTable The {@link KeyedTable} to scan. * @param filters Optional list of filters to apply to the scan. * @param splitCount The atomic integer to track the split count. - * @return The list of planned {@link ArcticSplit} included {@link MergeOnReadSplit}. + * @return The list of planned {@link AmoroSplit} included {@link MergeOnReadSplit}. */ - public static List mergeOnReadPlan( + public static List mergeOnReadPlan( KeyedTable keyedTable, List filters, AtomicInteger splitCount) { KeyedTableScan keyedTableScan = keyedTable.newScan(); if (filters != null) { filters.forEach(keyedTableScan::filter); } CloseableIterable combinedScanTasks = keyedTableScan.planTasks(); - List morSplits = Lists.newArrayList(); + List morSplits = Lists.newArrayList(); try (CloseableIterator initTasks = combinedScanTasks.iterator()) { while (initTasks.hasNext()) { @@ -137,16 +137,16 @@ public static List mergeOnReadPlan( return morSplits; } - public static List planChangeTable( + public static List planChangeTable( ChangeTableIncrementalScan tableIncrementalScan, AtomicInteger splitCount) { CloseableIterable tasks = tableIncrementalScan.planFiles(); BaseAndChangeTask baseAndChangeTask = BaseAndChangeTask.ofIceberg(tasks); return planChangeTable(baseAndChangeTask.transactionTasks(), splitCount); } - private static List planChangeTable( + private static List planChangeTable( Collection transactionTasks, AtomicInteger splitCount) { - List changeTasks = new ArrayList<>(transactionTasks.size()); + List changeTasks = new ArrayList<>(transactionTasks.size()); transactionTasks.forEach( transactionTask -> { PartitionAndNodeGroup partitionAndNodeGroup = diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/MixedIncrementalLoader.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/MixedIncrementalLoader.java index abe966f803..d5135158aa 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/MixedIncrementalLoader.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/MixedIncrementalLoader.java @@ -18,11 +18,11 @@ package org.apache.amoro.flink.read; -import org.apache.amoro.flink.read.hybrid.enumerator.ArcticEnumeratorOffset; +import org.apache.amoro.flink.read.hybrid.enumerator.AmoroEnumeratorOffset; import org.apache.amoro.flink.read.hybrid.enumerator.ContinuousEnumerationResult; import org.apache.amoro.flink.read.hybrid.enumerator.ContinuousSplitPlanner; import org.apache.amoro.flink.read.hybrid.reader.DataIteratorReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.hive.io.reader.AbstractAdaptHiveKeyedDataReader; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterator; @@ -52,8 +52,8 @@ public class MixedIncrementalLoader implements AutoCloseable { private final DataIteratorReaderFunction readerFunction; private AbstractAdaptHiveKeyedDataReader flinkArcticMORDataReader; private final List filters; - private final AtomicReference enumeratorPosition; - private final Queue splitQueue; + private final AtomicReference enumeratorPosition; + private final Queue splitQueue; public MixedIncrementalLoader( ContinuousSplitPlanner continuousSplitPlanner, @@ -100,7 +100,7 @@ public boolean hasNext() { } public CloseableIterator next() { - ArcticSplit split = splitQueue.poll(); + AmoroSplit split = splitQueue.poll(); if (split == null) { throw new IllegalStateException("next() called, but no more valid splits"); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/PartitionAndNodeGroup.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/PartitionAndNodeGroup.java index be136e7228..cfecc60cfe 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/PartitionAndNodeGroup.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/PartitionAndNodeGroup.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.scan.MixedFileScanTask; @@ -32,7 +32,7 @@ /** * This is a group of the partitions and nodes of the arctic table, it can plan different nodes and - * different partitions into different {@link ArcticSplit}. + * different partitions into different {@link AmoroSplit}. */ public class PartitionAndNodeGroup { AtomicInteger splitCount = new AtomicInteger(); @@ -54,12 +54,12 @@ public PartitionAndNodeGroup splitCount(AtomicInteger splitCount) { return this; } - List planSplits() { + List planSplits() { Map> nodes = new HashMap<>(); plan(true, nodes); plan(false, nodes); - List splits = new ArrayList<>(); + List splits = new ArrayList<>(); nodes .values() diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/ShuffleSplitAssigner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/ShuffleSplitAssigner.java index 5ae575f3fa..09d01ab6c9 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/ShuffleSplitAssigner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/ShuffleSplitAssigner.java @@ -20,9 +20,9 @@ import org.apache.amoro.data.DataTreeNode; import org.apache.amoro.data.PrimaryKeyedFile; -import org.apache.amoro.flink.read.hybrid.enumerator.ArcticSourceEnumState; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.enumerator.AmoroSourceEnumState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.amoro.scan.MixedFileScanTask; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.connector.source.SplitEnumeratorContext; @@ -54,7 +54,7 @@ public class ShuffleSplitAssigner implements SplitAssigner { private static final Logger LOG = LoggerFactory.getLogger(ShuffleSplitAssigner.class); private static final long POLL_TIMEOUT = 200; - private final SplitEnumeratorContext enumeratorContext; + private final SplitEnumeratorContext enumeratorContext; private int totalParallelism; private int totalSplitNum; @@ -67,12 +67,12 @@ public class ShuffleSplitAssigner implements SplitAssigner { */ private final Map partitionIndexSubtaskMap; /** Key is subtaskId, Value is the queue of unAssigned arctic splits. */ - private final Map> subtaskSplitMap; + private final Map> subtaskSplitMap; private CompletableFuture availableFuture; @VisibleForTesting - public ShuffleSplitAssigner(SplitEnumeratorContext enumeratorContext) { + public ShuffleSplitAssigner(SplitEnumeratorContext enumeratorContext) { this.enumeratorContext = enumeratorContext; this.totalParallelism = enumeratorContext.currentParallelism(); this.partitionIndexSubtaskMap = new ConcurrentHashMap<>(); @@ -80,9 +80,9 @@ public ShuffleSplitAssigner(SplitEnumeratorContext enumeratorContex } public ShuffleSplitAssigner( - SplitEnumeratorContext enumeratorContext, + SplitEnumeratorContext enumeratorContext, String tableName, - @Nullable ArcticSourceEnumState enumState) { + @Nullable AmoroSourceEnumState enumState) { this.enumeratorContext = enumeratorContext; this.partitionIndexSubtaskMap = new ConcurrentHashMap<>(); this.subtaskSplitMap = new ConcurrentHashMap<>(); @@ -120,7 +120,7 @@ public Split getNext(int subtaskId) { .orElseGet(isEmpty() ? Split::unavailable : Split::subtaskUnavailable); } - private Optional getNextSplit(int subTaskId) { + private Optional getNextSplit(int subTaskId) { int currentParallelism = enumeratorContext.currentParallelism(); if (totalParallelism != currentParallelism) { throw new FlinkRuntimeException( @@ -129,9 +129,9 @@ private Optional getNextSplit(int subTaskId) { totalParallelism, currentParallelism)); } if (subtaskSplitMap.containsKey(subTaskId)) { - PriorityBlockingQueue queue = subtaskSplitMap.get(subTaskId); + PriorityBlockingQueue queue = subtaskSplitMap.get(subTaskId); - ArcticSplit arcticSplit = null; + AmoroSplit arcticSplit = null; try { arcticSplit = queue.poll(POLL_TIMEOUT, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { @@ -156,18 +156,18 @@ private Optional getNextSplit(int subTaskId) { } @Override - public void onDiscoveredSplits(Collection splits) { + public void onDiscoveredSplits(Collection splits) { splits.forEach(this::putArcticIntoQueue); // only complete pending future if new splits are discovered completeAvailableFuturesIfNeeded(); } @Override - public void onUnassignedSplits(Collection splits) { + public void onUnassignedSplits(Collection splits) { onDiscoveredSplits(splits); } - void putArcticIntoQueue(final ArcticSplit split) { + void putArcticIntoQueue(final AmoroSplit split) { List exactlyTreeNodes = getExactlyTreeNodes(split); PrimaryKeyedFile file = findAnyFileInArcticSplit(split); @@ -184,9 +184,9 @@ void putArcticIntoQueue(final ArcticSplit split) { node.index(), subtaskId); - PriorityBlockingQueue queue = + PriorityBlockingQueue queue = subtaskSplitMap.getOrDefault(subtaskId, new PriorityBlockingQueue<>()); - ArcticSplit copiedSplit = split.copy(); + AmoroSplit copiedSplit = split.copy(); copiedSplit.modifyTreeNode(node); LOG.info("put split into queue: {}", copiedSplit); queue.add(copiedSplit); @@ -196,12 +196,12 @@ void putArcticIntoQueue(final ArcticSplit split) { } @Override - public Collection state() { - List arcticSplitStates = new ArrayList<>(); + public Collection state() { + List arcticSplitStates = new ArrayList<>(); subtaskSplitMap.forEach( (key, value) -> arcticSplitStates.addAll( - value.stream().map(ArcticSplitState::new).collect(Collectors.toList()))); + value.stream().map(AmoroSplitState::new).collect(Collectors.toList()))); return arcticSplitStates; } @@ -218,8 +218,7 @@ public boolean isEmpty() { if (subtaskSplitMap.isEmpty()) { return true; } - for (Map.Entry> entry : - subtaskSplitMap.entrySet()) { + for (Map.Entry> entry : subtaskSplitMap.entrySet()) { if (!entry.getValue().isEmpty()) { return false; } @@ -276,7 +275,7 @@ void deserializePartitionIndex(long[] shuffleSplitRelation) { * @param arcticSplit Arctic split. * @return The exact tree node list. */ - public List getExactlyTreeNodes(ArcticSplit arcticSplit) { + public List getExactlyTreeNodes(AmoroSplit arcticSplit) { DataTreeNode dataTreeNode = arcticSplit.dataTreeNode(); long mask = dataTreeNode.mask(); @@ -311,7 +310,7 @@ private List scanTreeNode(DataTreeNode dataTreeNode) { * @param arcticSplit arctic source split * @return anyone primary keyed file in the arcticSplit. */ - private PrimaryKeyedFile findAnyFileInArcticSplit(ArcticSplit arcticSplit) { + private PrimaryKeyedFile findAnyFileInArcticSplit(AmoroSplit arcticSplit) { AtomicReference file = new AtomicReference<>(); if (arcticSplit.isChangelogSplit()) { List arcticSplits = diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/Split.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/Split.java index 69309dc856..fe6f120219 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/Split.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/Split.java @@ -18,11 +18,11 @@ package org.apache.amoro.flink.read.hybrid.assigner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.util.Preconditions; -/** This is a wrapper Split of {@link ArcticSplit} with split status. */ +/** This is a wrapper Split of {@link AmoroSplit} with split status. */ public class Split { public enum Status { @@ -36,14 +36,14 @@ public enum Status { } private final Status status; - private final ArcticSplit split; + private final AmoroSplit split; private Split(Status status) { this.status = status; this.split = null; } - private Split(ArcticSplit split) { + private Split(AmoroSplit split) { Preconditions.checkNotNull(split, "Split cannot be null"); this.status = Status.AVAILABLE; this.split = split; @@ -62,7 +62,7 @@ public boolean isUnavailable() { return status == Status.UNAVAILABLE; } - public ArcticSplit split() { + public AmoroSplit split() { return split; } @@ -77,7 +77,7 @@ public static Split subtaskUnavailable() { return SUBTASK_UNAVAILABLE; } - public static Split of(ArcticSplit arcticSplit) { + public static Split of(AmoroSplit arcticSplit) { return new Split(arcticSplit); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/SplitAssigner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/SplitAssigner.java index d446fd3357..e6ae34b982 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/SplitAssigner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/SplitAssigner.java @@ -18,15 +18,15 @@ package org.apache.amoro.flink.read.hybrid.assigner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.flink.api.connector.source.SplitEnumeratorContext; import java.io.Closeable; import java.util.Collection; import java.util.concurrent.CompletableFuture; -/** An interface SplitAssigner for {@link ArcticSplit} */ +/** An interface SplitAssigner for {@link AmoroSplit} */ public interface SplitAssigner extends Closeable { default void open() {} @@ -36,10 +36,10 @@ default void open() {} Split getNext(int subtaskId); /** Add new splits discovered by enumerator */ - void onDiscoveredSplits(Collection splits); + void onDiscoveredSplits(Collection splits); /** Forward addSplitsBack event (for failed reader) to assigner */ - void onUnassignedSplits(Collection splits); + void onUnassignedSplits(Collection splits); /** * Some assigner (like event time alignment) may rack in-progress splits to advance watermark upon @@ -47,7 +47,7 @@ default void open() {} */ default void onCompletedSplits(Collection completedSplitIds) {} - Collection state(); + Collection state(); /** * Enumerator can get a notification via CompletableFuture when the assigner has more splits diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/StaticSplitAssigner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/StaticSplitAssigner.java index 91fa90bf18..aeb8833ee5 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/StaticSplitAssigner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/assigner/StaticSplitAssigner.java @@ -18,9 +18,9 @@ package org.apache.amoro.flink.read.hybrid.assigner; -import org.apache.amoro.flink.read.hybrid.enumerator.ArcticSourceEnumState; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.enumerator.AmoroSourceEnumState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,14 +42,14 @@ public class StaticSplitAssigner implements SplitAssigner { private static final long POLL_TIMEOUT = 200; private int totalSplitNum; - private final PriorityBlockingQueue splitQueue; + private final PriorityBlockingQueue splitQueue; private CompletableFuture availableFuture; - public StaticSplitAssigner(@Nullable ArcticSourceEnumState enumState) { + public StaticSplitAssigner(@Nullable AmoroSourceEnumState enumState) { this.splitQueue = new PriorityBlockingQueue<>(); if (enumState != null) { - Collection splitStates = enumState.pendingSplits(); + Collection splitStates = enumState.pendingSplits(); splitStates.forEach( state -> onDiscoveredSplits(Collections.singleton(state.toSourceSplit()))); } @@ -65,8 +65,8 @@ public Split getNext(int subtaskId) { return getNext(); } - private Optional getNextSplit() { - ArcticSplit arcticSplit = null; + private Optional getNextSplit() { + AmoroSplit arcticSplit = null; try { arcticSplit = splitQueue.poll(POLL_TIMEOUT, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { @@ -86,7 +86,7 @@ private Optional getNextSplit() { } @Override - public void onDiscoveredSplits(Collection splits) { + public void onDiscoveredSplits(Collection splits) { splits.forEach(this::putArcticIntoQueue); totalSplitNum += splits.size(); // only complete pending future if new splits are discovered @@ -94,17 +94,17 @@ public void onDiscoveredSplits(Collection splits) { } @Override - public void onUnassignedSplits(Collection splits) { + public void onUnassignedSplits(Collection splits) { onDiscoveredSplits(splits); } - void putArcticIntoQueue(final ArcticSplit split) { + void putArcticIntoQueue(final AmoroSplit split) { splitQueue.put(split); } @Override - public Collection state() { - return splitQueue.stream().map(ArcticSplitState::new).collect(Collectors.toList()); + public Collection state() { + return splitQueue.stream().map(AmoroSplitState::new).collect(Collectors.toList()); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AbstractArcticEnumerator.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AbstractAmoroEnumerator.java similarity index 92% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AbstractArcticEnumerator.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AbstractAmoroEnumerator.java index b151e02ad9..82fffeca3d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AbstractArcticEnumerator.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AbstractAmoroEnumerator.java @@ -21,7 +21,7 @@ import org.apache.amoro.flink.read.hybrid.assigner.Split; import org.apache.amoro.flink.read.hybrid.assigner.SplitAssigner; import org.apache.amoro.flink.read.hybrid.reader.ReaderStartedEvent; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.SplitRequestEvent; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.connector.source.SourceEvent; @@ -41,16 +41,16 @@ import java.util.concurrent.atomic.AtomicReference; /** The abstract arctic source enumerator. */ -public abstract class AbstractArcticEnumerator - implements SplitEnumerator { - private static final Logger LOG = LoggerFactory.getLogger(AbstractArcticEnumerator.class); - private final SplitEnumeratorContext enumeratorContext; +public abstract class AbstractAmoroEnumerator + implements SplitEnumerator { + private static final Logger LOG = LoggerFactory.getLogger(AbstractAmoroEnumerator.class); + private final SplitEnumeratorContext enumeratorContext; private final SplitAssigner assigner; private final Map readersAwaitingSplit; private final AtomicReference> availableFuture; - AbstractArcticEnumerator( - SplitEnumeratorContext enumeratorContext, SplitAssigner assigner) { + AbstractAmoroEnumerator( + SplitEnumeratorContext enumeratorContext, SplitAssigner assigner) { this.enumeratorContext = enumeratorContext; this.assigner = assigner; this.readersAwaitingSplit = new ConcurrentHashMap<>(); @@ -98,7 +98,7 @@ public void addReader(int subtaskId) { } @Override - public void addSplitsBack(List splits, int subtaskId) { + public void addSplitsBack(List splits, int subtaskId) { LOG.info("addSplitsBack from subtaskId {}, splits {}.", subtaskId, splits); assigner.onUnassignedSplits(splits); } @@ -130,7 +130,7 @@ protected void assignSplits() { final int awaitingSubtask = nextAwaiting.getKey(); final Split nextSplit = assigner.getNext(awaitingSubtask); if (nextSplit.isAvailable()) { - ArcticSplit arcticSplit = nextSplit.split(); + AmoroSplit arcticSplit = nextSplit.split(); LOG.info( "assign a arctic split to subtaskId {}, taskIndex {}, arcticSplit {}.", awaitingSubtask, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticEnumeratorOffset.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroEnumeratorOffset.java similarity index 83% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticEnumeratorOffset.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroEnumeratorOffset.java index 382e63a246..94235f9bce 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticEnumeratorOffset.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroEnumeratorOffset.java @@ -24,8 +24,8 @@ /** * The enumerator offset indicate the snapshot id of the change table, or the timestamp of snapshot. */ -public class ArcticEnumeratorOffset { - private static final ArcticEnumeratorOffset EMPTY = of(Long.MIN_VALUE, Long.MIN_VALUE); +public class AmoroEnumeratorOffset { + private static final AmoroEnumeratorOffset EMPTY = of(Long.MIN_VALUE, Long.MIN_VALUE); /** use Long.MIN_VALUE to indicate the earliest offset */ public static final long EARLIEST_SNAPSHOT_ID = Long.MIN_VALUE; @@ -33,16 +33,16 @@ public class ArcticEnumeratorOffset { private Long changeSnapshotId; private Long snapshotTimestampMs; - private ArcticEnumeratorOffset(Long changeSnapshotId, Long snapshotTimestampMs) { + private AmoroEnumeratorOffset(Long changeSnapshotId, Long snapshotTimestampMs) { this.changeSnapshotId = changeSnapshotId; this.snapshotTimestampMs = snapshotTimestampMs; } - public static ArcticEnumeratorOffset of(Long changeSnapshotId, Long snapshotTimestampMs) { - return new ArcticEnumeratorOffset(changeSnapshotId, snapshotTimestampMs); + public static AmoroEnumeratorOffset of(Long changeSnapshotId, Long snapshotTimestampMs) { + return new AmoroEnumeratorOffset(changeSnapshotId, snapshotTimestampMs); } - public static ArcticEnumeratorOffset empty() { + public static AmoroEnumeratorOffset empty() { return EMPTY; } @@ -87,7 +87,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - ArcticEnumeratorOffset other = (ArcticEnumeratorOffset) o; + AmoroEnumeratorOffset other = (AmoroEnumeratorOffset) o; return Objects.equal(changeSnapshotId, other.changeSnapshotId()) && Objects.equal(snapshotTimestampMs, other.snapshotTimestampMs()); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticEnumeratorOffsetSerializer.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroEnumeratorOffsetSerializer.java similarity index 77% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticEnumeratorOffsetSerializer.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroEnumeratorOffsetSerializer.java index 262c1e1306..c1ba3e6e54 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticEnumeratorOffsetSerializer.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroEnumeratorOffsetSerializer.java @@ -24,11 +24,10 @@ import java.io.IOException; -/** Serializer that serializes and deserializes arctic enumerator {@link ArcticEnumeratorOffset}. */ -class ArcticEnumeratorOffsetSerializer - implements SimpleVersionedSerializer { - public static final ArcticEnumeratorOffsetSerializer INSTANCE = - new ArcticEnumeratorOffsetSerializer(); +/** Serializer that serializes and deserializes arctic enumerator {@link AmoroEnumeratorOffset}. */ +class AmoroEnumeratorOffsetSerializer implements SimpleVersionedSerializer { + public static final AmoroEnumeratorOffsetSerializer INSTANCE = + new AmoroEnumeratorOffsetSerializer(); private static final int VERSION = 1; @@ -41,12 +40,12 @@ public int getVersion() { } @Override - public byte[] serialize(ArcticEnumeratorOffset position) throws IOException { + public byte[] serialize(AmoroEnumeratorOffset position) throws IOException { return serializeV1(position); } @Override - public ArcticEnumeratorOffset deserialize(int version, byte[] serialized) throws IOException { + public AmoroEnumeratorOffset deserialize(int version, byte[] serialized) throws IOException { switch (version) { case 1: return deserializeV1(serialized); @@ -55,7 +54,7 @@ public ArcticEnumeratorOffset deserialize(int version, byte[] serialized) throws } } - private byte[] serializeV1(ArcticEnumeratorOffset position) throws IOException { + private byte[] serializeV1(AmoroEnumeratorOffset position) throws IOException { DataOutputSerializer out = SERIALIZER_CACHE.get(); out.writeBoolean(position.changeSnapshotId() != null); if (position.changeSnapshotId() != null) { @@ -70,7 +69,7 @@ private byte[] serializeV1(ArcticEnumeratorOffset position) throws IOException { return result; } - private ArcticEnumeratorOffset deserializeV1(byte[] serialized) throws IOException { + private AmoroEnumeratorOffset deserializeV1(byte[] serialized) throws IOException { DataInputDeserializer in = new DataInputDeserializer(serialized); Long snapshotId = null; if (in.readBoolean()) { @@ -82,6 +81,6 @@ private ArcticEnumeratorOffset deserializeV1(byte[] serialized) throws IOExcepti snapshotTimestampMs = in.readLong(); } - return ArcticEnumeratorOffset.of(snapshotId, snapshotTimestampMs); + return AmoroEnumeratorOffset.of(snapshotId, snapshotTimestampMs); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumState.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumState.java similarity index 77% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumState.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumState.java index a3f8c8a56a..945adf1ad8 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumState.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumState.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; import javax.annotation.Nullable; @@ -27,17 +27,17 @@ /** * State that contains pending arctic splits and last enumerator offset in arctic source enumerator - * {@link ArcticSourceEnumerator}. + * {@link AmoroSourceEnumerator}. */ -public class ArcticSourceEnumState { - @Nullable private final ArcticEnumeratorOffset lastEnumeratedOffset; - private final Collection pendingSplits; +public class AmoroSourceEnumState { + @Nullable private final AmoroEnumeratorOffset lastEnumeratedOffset; + private final Collection pendingSplits; @Nullable private final long[] shuffleSplitRelation; @Nullable private final TemporalJoinSplits temporalJoinSplits; - public ArcticSourceEnumState( - Collection pendingSplits, - @Nullable ArcticEnumeratorOffset lastEnumeratedOffset, + public AmoroSourceEnumState( + Collection pendingSplits, + @Nullable AmoroEnumeratorOffset lastEnumeratedOffset, @Nullable long[] shuffleSplitRelation, @Nullable TemporalJoinSplits temporalJoinSplits) { this.pendingSplits = pendingSplits; @@ -47,11 +47,11 @@ public ArcticSourceEnumState( } @Nullable - public ArcticEnumeratorOffset lastEnumeratedOffset() { + public AmoroEnumeratorOffset lastEnumeratedOffset() { return lastEnumeratedOffset; } - public Collection pendingSplits() { + public Collection pendingSplits() { return pendingSplits; } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumStateSerializer.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumStateSerializer.java similarity index 77% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumStateSerializer.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumStateSerializer.java index 81047cc88a..c591dd66a7 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumStateSerializer.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumStateSerializer.java @@ -18,9 +18,9 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitSerializer; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitSerializer; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; import org.apache.flink.core.io.SimpleVersionedSerializer; import org.apache.flink.core.memory.DataInputDeserializer; @@ -34,16 +34,16 @@ import java.util.Collection; import java.util.Objects; -/** Serializer that serializes and deserializes arctic enumerator {@link ArcticSourceEnumState}. */ -public class ArcticSourceEnumStateSerializer - implements SimpleVersionedSerializer { +/** Serializer that serializes and deserializes arctic enumerator {@link AmoroSourceEnumState}. */ +public class AmoroSourceEnumStateSerializer + implements SimpleVersionedSerializer { private static final Logger LOGGER = - LoggerFactory.getLogger(ArcticSourceEnumStateSerializer.class); + LoggerFactory.getLogger(AmoroSourceEnumStateSerializer.class); private static final int VERSION = 1; - private final ArcticSplitSerializer splitSerializer = ArcticSplitSerializer.INSTANCE; - private final ArcticEnumeratorOffsetSerializer offsetSerializer = - ArcticEnumeratorOffsetSerializer.INSTANCE; + private final AmoroSplitSerializer splitSerializer = AmoroSplitSerializer.INSTANCE; + private final AmoroEnumeratorOffsetSerializer offsetSerializer = + AmoroEnumeratorOffsetSerializer.INSTANCE; private static final ThreadLocal SERIALIZER_CACHE = ThreadLocal.withInitial(() -> new DataOutputSerializer(1024)); @@ -54,11 +54,11 @@ public int getVersion() { } @Override - public byte[] serialize(ArcticSourceEnumState arcticSourceEnumState) throws IOException { + public byte[] serialize(AmoroSourceEnumState arcticSourceEnumState) throws IOException { return serializeV1(arcticSourceEnumState); } - private byte[] serializeV1(ArcticSourceEnumState enumState) throws IOException { + private byte[] serializeV1(AmoroSourceEnumState enumState) throws IOException { DataOutputSerializer out = SERIALIZER_CACHE.get(); out.writeBoolean(enumState.lastEnumeratedOffset() != null); @@ -71,7 +71,7 @@ private byte[] serializeV1(ArcticSourceEnumState enumState) throws IOException { out.writeInt(splitSerializer.getVersion()); out.writeInt(enumState.pendingSplits().size()); - for (ArcticSplitState splitState : enumState.pendingSplits()) { + for (AmoroSplitState splitState : enumState.pendingSplits()) { byte[] splitBytes = splitSerializer.serialize(splitState.toSourceSplit()); out.writeInt(splitBytes.length); out.write(splitBytes); @@ -99,7 +99,7 @@ private byte[] serializeV1(ArcticSourceEnumState enumState) throws IOException { } @Override - public ArcticSourceEnumState deserialize(int version, byte[] serialized) throws IOException { + public AmoroSourceEnumState deserialize(int version, byte[] serialized) throws IOException { switch (version) { case 1: return deserializeV1(serialized); @@ -108,10 +108,10 @@ public ArcticSourceEnumState deserialize(int version, byte[] serialized) throws } } - private ArcticSourceEnumState deserializeV1(byte[] serialized) throws IOException { + private AmoroSourceEnumState deserializeV1(byte[] serialized) throws IOException { DataInputDeserializer in = new DataInputDeserializer(serialized); - ArcticEnumeratorOffset enumeratorOffset = null; + AmoroEnumeratorOffset enumeratorOffset = null; if (in.readBoolean()) { int version = in.readInt(); byte[] positionBytes = new byte[in.readInt()]; @@ -121,12 +121,12 @@ private ArcticSourceEnumState deserializeV1(byte[] serialized) throws IOExceptio int splitSerializerVersion = in.readInt(); int splitCount = in.readInt(); - Collection pendingSplits = Lists.newArrayListWithCapacity(splitCount); + Collection pendingSplits = Lists.newArrayListWithCapacity(splitCount); for (int i = 0; i < splitCount; ++i) { byte[] splitBytes = new byte[in.readInt()]; in.read(splitBytes); - ArcticSplit split = splitSerializer.deserialize(splitSerializerVersion, splitBytes); - pendingSplits.add(new ArcticSplitState(split)); + AmoroSplit split = splitSerializer.deserialize(splitSerializerVersion, splitBytes); + pendingSplits.add(new AmoroSplitState(split)); } long[] shuffleSplitRelation = null; @@ -150,7 +150,7 @@ private ArcticSourceEnumState deserializeV1(byte[] serialized) throws IOExceptio } } - return new ArcticSourceEnumState( + return new AmoroSourceEnumState( pendingSplits, enumeratorOffset, shuffleSplitRelation, temporalJoinSplits); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumerator.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumerator.java similarity index 84% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumerator.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumerator.java index 577ec4ea30..566f9d21fb 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ArcticSourceEnumerator.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/AmoroSourceEnumerator.java @@ -18,21 +18,21 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import static org.apache.amoro.flink.read.hybrid.enumerator.ArcticEnumeratorOffset.EARLIEST_SNAPSHOT_ID; +import static org.apache.amoro.flink.read.hybrid.enumerator.AmoroEnumeratorOffset.EARLIEST_SNAPSHOT_ID; import static org.apache.amoro.flink.table.descriptors.ArcticValidator.SCAN_STARTUP_MODE; import static org.apache.amoro.flink.table.descriptors.ArcticValidator.SCAN_STARTUP_MODE_LATEST; -import static org.apache.amoro.flink.util.ArcticUtils.loadArcticTable; +import static org.apache.amoro.flink.util.AmoroUtils.loadArcticTable; import org.apache.amoro.flink.read.hybrid.assigner.ShuffleSplitAssigner; import org.apache.amoro.flink.read.hybrid.assigner.SplitAssigner; -import org.apache.amoro.flink.read.hybrid.reader.ArcticSourceReader; +import org.apache.amoro.flink.read.hybrid.reader.AmoroSourceReader; import org.apache.amoro.flink.read.hybrid.reader.HybridSplitReader; import org.apache.amoro.flink.read.hybrid.reader.ReaderStartedEvent; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.SplitRequestEvent; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; -import org.apache.amoro.flink.read.source.ArcticScanContext; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.read.source.AmoroScanContext; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.table.KeyedTable; import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SplitEnumeratorContext; @@ -51,11 +51,11 @@ import java.util.stream.IntStream; /** - * Enumerator for arctic source, assign {@link ArcticSplit} to arctic source reader {@link + * Enumerator for arctic source, assign {@link AmoroSplit} to arctic source reader {@link * HybridSplitReader} */ -public class ArcticSourceEnumerator extends AbstractArcticEnumerator { - private static final Logger LOG = LoggerFactory.getLogger(ArcticSourceEnumerator.class); +public class AmoroSourceEnumerator extends AbstractAmoroEnumerator { + private static final Logger LOG = LoggerFactory.getLogger(AmoroSourceEnumerator.class); private transient KeyedTable keyedTable; /** * To record the snapshotId at the first planSplits. @@ -64,16 +64,16 @@ public class ArcticSourceEnumerator extends AbstractArcticEnumerator { */ private transient volatile TemporalJoinSplits temporalJoinSplits = null; - private final ArcticTableLoader loader; - private final SplitEnumeratorContext context; + private final AmoroTableLoader loader; + private final SplitEnumeratorContext context; private final ContinuousSplitPlanner continuousSplitPlanner; private final SplitAssigner splitAssigner; - private final ArcticScanContext scanContext; + private final AmoroScanContext scanContext; private final long snapshotDiscoveryIntervalMs; /** - * If true, using arctic table as build table. {@link ArcticSourceEnumerator} will notify {@link - * ArcticSourceReader} after ArcticReaders have finished reading all {@link TemporalJoinSplits}. - * Then {@link ArcticSourceReader} will emit a Watermark values Long.MAX_VALUE. Advancing + * If true, using arctic table as build table. {@link AmoroSourceEnumerator} will notify {@link + * AmoroSourceReader} after ArcticReaders have finished reading all {@link TemporalJoinSplits}. + * Then {@link AmoroSourceReader} will emit a Watermark values Long.MAX_VALUE. Advancing * TemporalJoinOperator's watermark can trigger the join operation and push the results to * downstream. The watermark of Long.MAX_VALUE avoids affecting the watermark defined by user * arbitrary probe side @@ -85,16 +85,16 @@ public class ArcticSourceEnumerator extends AbstractArcticEnumerator { * snapshotId for the last enumerated snapshot. next incremental enumeration should be based off * this as the starting position. */ - private final AtomicReference enumeratorPosition; + private final AtomicReference enumeratorPosition; private final AtomicBoolean lock = new AtomicBoolean(false); - public ArcticSourceEnumerator( - SplitEnumeratorContext enumContext, + public AmoroSourceEnumerator( + SplitEnumeratorContext enumContext, SplitAssigner splitAssigner, - ArcticTableLoader loader, - ArcticScanContext scanContext, - @Nullable ArcticSourceEnumState enumState, + AmoroTableLoader loader, + AmoroScanContext scanContext, + @Nullable AmoroSourceEnumState enumState, boolean dimTable) { super(enumContext, splitAssigner); this.loader = loader; @@ -122,7 +122,7 @@ public void start() { keyedTable.refresh(); Snapshot snapshot = keyedTable.changeTable().currentSnapshot(); long snapshotId = snapshot == null ? EARLIEST_SNAPSHOT_ID : snapshot.snapshotId(); - enumeratorPosition.set(ArcticEnumeratorOffset.of(snapshotId, null)); + enumeratorPosition.set(AmoroEnumeratorOffset.of(snapshotId, null)); LOG.info( "{} is {}, the current snapshot id of the change table {} is {}.", SCAN_STARTUP_MODE.key(), @@ -214,7 +214,7 @@ public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { /** * Check whether all first splits have been finished or not. After all finished, enumerator will - * send a {@link InitializationFinishedEvent} to notify all {@link ArcticSourceReader}. + * send a {@link InitializationFinishedEvent} to notify all {@link AmoroSourceReader}. * * @param finishedSplitIds */ @@ -240,12 +240,12 @@ private void notifyReaders() { } @Override - public ArcticSourceEnumState snapshotState(long checkpointId) throws Exception { + public AmoroSourceEnumState snapshotState(long checkpointId) throws Exception { long[] shuffleSplitRelation = null; if (splitAssigner instanceof ShuffleSplitAssigner) { shuffleSplitRelation = ((ShuffleSplitAssigner) splitAssigner).serializePartitionIndex(); } - return new ArcticSourceEnumState( + return new AmoroSourceEnumState( splitAssigner.state(), enumeratorPosition.get(), shuffleSplitRelation, temporalJoinSplits); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousEnumerationResult.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousEnumerationResult.java index b87fd93c16..f5e90f73dd 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousEnumerationResult.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousEnumerationResult.java @@ -19,7 +19,7 @@ package org.apache.amoro.flink.read.hybrid.enumerator; import org.apache.amoro.flink.read.FlinkSplitPlanner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -27,15 +27,14 @@ import java.util.Collection; import java.util.Collections; -/** The result that contains {@link ArcticSplit}s and is generated by {@link FlinkSplitPlanner}. */ +/** The result that contains {@link AmoroSplit}s and is generated by {@link FlinkSplitPlanner}. */ public class ContinuousEnumerationResult { public static final ContinuousEnumerationResult EMPTY = - new ContinuousEnumerationResult( - Collections.emptyList(), null, ArcticEnumeratorOffset.empty()); + new ContinuousEnumerationResult(Collections.emptyList(), null, AmoroEnumeratorOffset.empty()); - private final Collection splits; - private final ArcticEnumeratorOffset fromOffset; - private final ArcticEnumeratorOffset toOffset; + private final Collection splits; + private final AmoroEnumeratorOffset fromOffset; + private final AmoroEnumeratorOffset toOffset; /** * @param splits should never be null. But it can be an empty collection @@ -43,9 +42,9 @@ public class ContinuousEnumerationResult { * @param toOffset should never be null. But it can have null snapshotId and snapshotTimestampMs */ public ContinuousEnumerationResult( - Collection splits, - ArcticEnumeratorOffset fromOffset, - ArcticEnumeratorOffset toOffset) { + Collection splits, + AmoroEnumeratorOffset fromOffset, + AmoroEnumeratorOffset toOffset) { Preconditions.checkArgument(splits != null, "Invalid to splits collection: null"); Preconditions.checkArgument(toOffset != null, "Invalid end position: null"); this.splits = splits; @@ -53,15 +52,15 @@ public ContinuousEnumerationResult( this.toOffset = toOffset; } - public Collection splits() { + public Collection splits() { return splits; } - public ArcticEnumeratorOffset fromOffset() { + public AmoroEnumeratorOffset fromOffset() { return fromOffset; } - public ArcticEnumeratorOffset toOffset() { + public AmoroEnumeratorOffset toOffset() { return toOffset; } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlanner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlanner.java index 246e1b1341..89483febc5 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlanner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlanner.java @@ -30,7 +30,7 @@ public interface ContinuousSplitPlanner extends Closeable { /** Discover the files appended between {@code lastPosition} and current table snapshot */ - default ContinuousEnumerationResult planSplits(ArcticEnumeratorOffset lastPosition) { + default ContinuousEnumerationResult planSplits(AmoroEnumeratorOffset lastPosition) { return planSplits(lastPosition, Lists.newArrayList()); } @@ -39,5 +39,5 @@ default ContinuousEnumerationResult planSplits(ArcticEnumeratorOffset lastPositi * data with expressions. */ ContinuousEnumerationResult planSplits( - ArcticEnumeratorOffset lastPosition, List filters); + AmoroEnumeratorOffset lastPosition, List filters); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlannerImpl.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlannerImpl.java index f9ef222bac..4ceafa51e7 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlannerImpl.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/ContinuousSplitPlannerImpl.java @@ -19,14 +19,14 @@ package org.apache.amoro.flink.read.hybrid.enumerator; import static org.apache.amoro.flink.read.FlinkSplitPlanner.planChangeTable; -import static org.apache.amoro.flink.read.hybrid.enumerator.ArcticEnumeratorOffset.EARLIEST_SNAPSHOT_ID; -import static org.apache.amoro.flink.util.ArcticUtils.loadArcticTable; +import static org.apache.amoro.flink.read.hybrid.enumerator.AmoroEnumeratorOffset.EARLIEST_SNAPSHOT_ID; +import static org.apache.amoro.flink.util.AmoroUtils.loadArcticTable; import org.apache.amoro.flink.read.FlinkSplitPlanner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.flink.read.hybrid.split.SnapshotSplit; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.scan.ChangeTableIncrementalScan; import org.apache.amoro.table.KeyedTable; import org.apache.commons.collections.CollectionUtils; @@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** - * Continuous planning {@link KeyedTable} by {@link ArcticEnumeratorOffset} and generate a {@link + * Continuous planning {@link KeyedTable} by {@link AmoroEnumeratorOffset} and generate a {@link * ContinuousEnumerationResult}. * *

{@link ContinuousEnumerationResult#splits()} includes the {@link SnapshotSplit}s and {@link @@ -52,10 +52,10 @@ public class ContinuousSplitPlannerImpl implements ContinuousSplitPlanner { private static final Logger LOG = LoggerFactory.getLogger(ContinuousSplitPlannerImpl.class); protected transient KeyedTable table; - protected final ArcticTableLoader loader; + protected final AmoroTableLoader loader; protected static final AtomicInteger SPLIT_COUNT = new AtomicInteger(); - public ContinuousSplitPlannerImpl(ArcticTableLoader loader) { + public ContinuousSplitPlannerImpl(AmoroTableLoader loader) { this.loader = loader; } @@ -68,7 +68,7 @@ public void close() throws IOException { @Override public ContinuousEnumerationResult planSplits( - ArcticEnumeratorOffset lastOffset, List filters) { + AmoroEnumeratorOffset lastOffset, List filters) { if (table == null) { table = loadArcticTable(loader).asKeyedTable(); } @@ -81,7 +81,7 @@ public ContinuousEnumerationResult planSplits( } protected ContinuousEnumerationResult discoverIncrementalSplits( - ArcticEnumeratorOffset lastPosition, List filters) { + AmoroEnumeratorOffset lastPosition, List filters) { long fromChangeSnapshotId = lastPosition.changeSnapshotId(); Snapshot changeSnapshot = table.changeTable().currentSnapshot(); if (changeSnapshot != null && changeSnapshot.snapshotId() != fromChangeSnapshotId) { @@ -99,9 +99,9 @@ protected ContinuousEnumerationResult discoverIncrementalSplits( changeTableScan = changeTableScan.fromSequence(snapshot.sequenceNumber()); } - List arcticChangeSplit = planChangeTable(changeTableScan, SPLIT_COUNT); + List arcticChangeSplit = planChangeTable(changeTableScan, SPLIT_COUNT); return new ContinuousEnumerationResult( - arcticChangeSplit, lastPosition, ArcticEnumeratorOffset.of(snapshotId, null)); + arcticChangeSplit, lastPosition, AmoroEnumeratorOffset.of(snapshotId, null)); } return ContinuousEnumerationResult.EMPTY; } @@ -111,7 +111,7 @@ protected ContinuousEnumerationResult discoverInitialSplits(List fil // todo ShuffleSplitAssigner doesn't support MergeOnReadSplit right now, // because it doesn't implement the dataTreeNode() method // fix AMORO-1950 in the future. - List arcticSplits = FlinkSplitPlanner.planFullTable(table, filters, SPLIT_COUNT); + List arcticSplits = FlinkSplitPlanner.planFullTable(table, filters, SPLIT_COUNT); long changeStartSnapshotId = changeSnapshot != null ? changeSnapshot.snapshotId() : EARLIEST_SNAPSHOT_ID; @@ -121,6 +121,6 @@ protected ContinuousEnumerationResult discoverInitialSplits(List fil } return new ContinuousEnumerationResult( - arcticSplits, null, ArcticEnumeratorOffset.of(changeStartSnapshotId, null)); + arcticSplits, null, AmoroEnumeratorOffset.of(changeStartSnapshotId, null)); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/InitializationFinishedEvent.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/InitializationFinishedEvent.java index c03f7b3f0c..d96d0cf7fb 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/InitializationFinishedEvent.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/InitializationFinishedEvent.java @@ -18,10 +18,10 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import org.apache.amoro.flink.read.hybrid.reader.ArcticSourceReader; +import org.apache.amoro.flink.read.hybrid.reader.AmoroSourceReader; import org.apache.flink.api.connector.source.SourceEvent; -/** {@link ArcticSourceReader} won't set timestamp to RowData until receiving this Event. */ +/** {@link AmoroSourceReader} won't set timestamp to RowData until receiving this Event. */ public class InitializationFinishedEvent implements SourceEvent { private static final long serialVersionUID = 1L; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadIncrementalPlanner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadIncrementalPlanner.java index 45aabc21d8..a9a0997607 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadIncrementalPlanner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadIncrementalPlanner.java @@ -18,13 +18,13 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import static org.apache.amoro.flink.read.hybrid.enumerator.ArcticEnumeratorOffset.EARLIEST_SNAPSHOT_ID; +import static org.apache.amoro.flink.read.hybrid.enumerator.AmoroEnumeratorOffset.EARLIEST_SNAPSHOT_ID; import org.apache.amoro.flink.read.FlinkSplitPlanner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.flink.read.hybrid.split.MergeOnReadSplit; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.commons.collections.CollectionUtils; import org.apache.iceberg.Snapshot; import org.apache.iceberg.expressions.Expression; @@ -35,7 +35,7 @@ /** * A planner for merge-on-read scanning by {@link this#discoverInitialSplits} and incremental - * scanning by {@link this#discoverIncrementalSplits(ArcticEnumeratorOffset, List)}. + * scanning by {@link this#discoverIncrementalSplits(AmoroEnumeratorOffset, List)}. * *

{@link ContinuousEnumerationResult#splits()} includes the {@link MergeOnReadSplit}s and {@link * ChangelogSplit}s. @@ -43,7 +43,7 @@ public class MergeOnReadIncrementalPlanner extends ContinuousSplitPlannerImpl { private static final Logger LOG = LoggerFactory.getLogger(MergeOnReadIncrementalPlanner.class); - public MergeOnReadIncrementalPlanner(ArcticTableLoader loader) { + public MergeOnReadIncrementalPlanner(AmoroTableLoader loader) { super(loader); } @@ -51,7 +51,7 @@ public MergeOnReadIncrementalPlanner(ArcticTableLoader loader) { protected ContinuousEnumerationResult discoverInitialSplits(List filters) { Snapshot changeSnapshot = table.changeTable().currentSnapshot(); - List arcticSplits = FlinkSplitPlanner.mergeOnReadPlan(table, filters, SPLIT_COUNT); + List arcticSplits = FlinkSplitPlanner.mergeOnReadPlan(table, filters, SPLIT_COUNT); long changeStartSnapshotId = changeSnapshot != null ? changeSnapshot.snapshotId() : EARLIEST_SNAPSHOT_ID; @@ -61,6 +61,6 @@ protected ContinuousEnumerationResult discoverInitialSplits(List fil } return new ContinuousEnumerationResult( - arcticSplits, null, ArcticEnumeratorOffset.of(changeStartSnapshotId, null)); + arcticSplits, null, AmoroEnumeratorOffset.of(changeStartSnapshotId, null)); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadPlannerImpl.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadPlannerImpl.java index aa485a25be..83f757473c 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadPlannerImpl.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/MergeOnReadPlannerImpl.java @@ -18,12 +18,12 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import static org.apache.amoro.flink.read.hybrid.enumerator.ArcticEnumeratorOffset.EARLIEST_SNAPSHOT_ID; -import static org.apache.amoro.flink.util.ArcticUtils.loadArcticTable; +import static org.apache.amoro.flink.read.hybrid.enumerator.AmoroEnumeratorOffset.EARLIEST_SNAPSHOT_ID; +import static org.apache.amoro.flink.util.AmoroUtils.loadArcticTable; import org.apache.amoro.flink.read.FlinkSplitPlanner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.table.KeyedTable; import org.apache.commons.collections.CollectionUtils; import org.apache.iceberg.Snapshot; @@ -40,16 +40,16 @@ public class MergeOnReadPlannerImpl implements ContinuousSplitPlanner { private static final Logger LOG = LoggerFactory.getLogger(MergeOnReadPlannerImpl.class); protected transient KeyedTable table; - protected final ArcticTableLoader loader; + protected final AmoroTableLoader loader; protected static final AtomicInteger SPLIT_COUNT = new AtomicInteger(); - public MergeOnReadPlannerImpl(ArcticTableLoader loader) { + public MergeOnReadPlannerImpl(AmoroTableLoader loader) { this.loader = loader; } @Override public ContinuousEnumerationResult planSplits( - ArcticEnumeratorOffset ignored, List filters) { + AmoroEnumeratorOffset ignored, List filters) { // todo support mor the table from the specific offset in the future if (table == null) { table = loadArcticTable(loader).asKeyedTable(); @@ -60,7 +60,7 @@ public ContinuousEnumerationResult planSplits( protected ContinuousEnumerationResult discoverInitialSplits(List filters) { Snapshot changeSnapshot = table.changeTable().currentSnapshot(); - List arcticSplits = FlinkSplitPlanner.mergeOnReadPlan(table, filters, SPLIT_COUNT); + List arcticSplits = FlinkSplitPlanner.mergeOnReadPlan(table, filters, SPLIT_COUNT); long changeStartSnapshotId = changeSnapshot != null ? changeSnapshot.snapshotId() : EARLIEST_SNAPSHOT_ID; @@ -70,7 +70,7 @@ protected ContinuousEnumerationResult discoverInitialSplits(List fil } return new ContinuousEnumerationResult( - arcticSplits, null, ArcticEnumeratorOffset.of(changeStartSnapshotId, null)); + arcticSplits, null, AmoroEnumeratorOffset.of(changeStartSnapshotId, null)); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/StaticArcticSourceEnumerator.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/StaticAmoroSourceEnumerator.java similarity index 72% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/StaticArcticSourceEnumerator.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/StaticAmoroSourceEnumerator.java index 63e8776d17..7030952e76 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/StaticArcticSourceEnumerator.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/enumerator/StaticAmoroSourceEnumerator.java @@ -18,12 +18,12 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import static org.apache.amoro.flink.util.ArcticUtils.loadArcticTable; +import static org.apache.amoro.flink.util.AmoroUtils.loadArcticTable; import org.apache.amoro.flink.read.hybrid.assigner.SplitAssigner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.source.ArcticScanContext; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.source.AmoroScanContext; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.table.KeyedTable; import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.apache.iceberg.flink.source.ScanContext; @@ -38,21 +38,21 @@ * This is a static arctic source enumerator, used for bounded source scan. Working enabled only * just {@link ScanContext#STREAMING} is equal to false. */ -public class StaticArcticSourceEnumerator extends AbstractArcticEnumerator { - private static final Logger LOG = LoggerFactory.getLogger(StaticArcticSourceEnumerator.class); +public class StaticAmoroSourceEnumerator extends AbstractAmoroEnumerator { + private static final Logger LOG = LoggerFactory.getLogger(StaticAmoroSourceEnumerator.class); private final SplitAssigner assigner; - private final ArcticTableLoader loader; + private final AmoroTableLoader loader; private transient KeyedTable keyedTable; - private final ArcticScanContext scanContext; + private final AmoroScanContext scanContext; private final boolean shouldEnumerate; private final ContinuousSplitPlanner splitPlanner; - public StaticArcticSourceEnumerator( - SplitEnumeratorContext enumeratorContext, + public StaticAmoroSourceEnumerator( + SplitEnumeratorContext enumeratorContext, SplitAssigner assigner, - ArcticTableLoader loader, - ArcticScanContext scanContext, - @Nullable ArcticSourceEnumState enumState) { + AmoroTableLoader loader, + AmoroScanContext scanContext, + @Nullable AmoroSourceEnumState enumState) { super(enumeratorContext, assigner); this.loader = loader; this.assigner = assigner; @@ -71,8 +71,7 @@ public void start() { if (shouldEnumerate) { keyedTable.baseTable().refresh(); keyedTable.changeTable().refresh(); - Collection splits = - splitPlanner.planSplits(null, scanContext.filters()).splits(); + Collection splits = splitPlanner.planSplits(null, scanContext.filters()).splits(); assigner.onDiscoveredSplits(splits); LOG.info( "Discovered {} splits from table {} during job initialization", @@ -87,7 +86,7 @@ protected boolean shouldWaitForMoreSplits() { } @Override - public ArcticSourceEnumState snapshotState(long checkpointId) throws Exception { - return new ArcticSourceEnumState(assigner.state(), null, null, null); + public AmoroSourceEnumState snapshotState(long checkpointId) throws Exception { + return new AmoroSourceEnumState(assigner.state(), null, null, null); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticRecordEmitter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroRecordEmitter.java similarity index 85% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticRecordEmitter.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroRecordEmitter.java index 61ee5c835e..d7fabb772e 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticRecordEmitter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroRecordEmitter.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read.hybrid.reader; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.flink.api.connector.source.SourceOutput; import org.apache.flink.connector.base.source.reader.RecordEmitter; import org.apache.flink.table.data.GenericRowData; @@ -33,21 +33,21 @@ * Emitter that emit {@link T} to the next flink operator and update the record offset of {@link T} * into split state. */ -public class ArcticRecordEmitter - implements RecordEmitter, T, ArcticSplitState> { +public class AmoroRecordEmitter + implements RecordEmitter, T, AmoroSplitState> { - public static final Logger LOGGER = LoggerFactory.getLogger(ArcticRecordEmitter.class); + public static final Logger LOGGER = LoggerFactory.getLogger(AmoroRecordEmitter.class); /** It signifies whether the Long.MIN_VALUE need to be set into RowData. */ public boolean populateRowTime; - public ArcticRecordEmitter(boolean populateRowTime) { + public AmoroRecordEmitter(boolean populateRowTime) { this.populateRowTime = populateRowTime; } @Override public void emitRecord( - ArcticRecordWithOffset element, SourceOutput sourceOutput, ArcticSplitState split) + AmoroRecordWithOffset element, SourceOutput sourceOutput, AmoroSplitState split) throws Exception { T record = element.record(); if (!populateRowTime) { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticRecordWithOffset.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroRecordWithOffset.java similarity index 97% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticRecordWithOffset.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroRecordWithOffset.java index cb89c8dfa3..805c26d569 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticRecordWithOffset.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroRecordWithOffset.java @@ -19,7 +19,7 @@ package org.apache.amoro.flink.read.hybrid.reader; /** A record along with the reader position to be stored in the checkpoint. */ -public class ArcticRecordWithOffset { +public class AmoroRecordWithOffset { private T record; private int insertFileOffset; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticSourceReader.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroSourceReader.java similarity index 87% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticSourceReader.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroSourceReader.java index 38268013e3..62da51e1db 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArcticSourceReader.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/AmoroSourceReader.java @@ -18,10 +18,10 @@ package org.apache.amoro.flink.read.hybrid.reader; -import org.apache.amoro.flink.read.ArcticSource; +import org.apache.amoro.flink.read.AmoroSource; import org.apache.amoro.flink.read.hybrid.enumerator.InitializationFinishedEvent; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.amoro.flink.read.hybrid.split.SplitRequestEvent; import org.apache.amoro.flink.util.FlinkClassReflectionUtil; import org.apache.flink.api.common.eventtime.Watermark; @@ -45,26 +45,26 @@ import java.util.Map; /** - * Arctic source reader that is created by a {@link ArcticSource#createReader(SourceReaderContext)}. + * Arctic source reader that is created by a {@link AmoroSource#createReader(SourceReaderContext)}. */ -public class ArcticSourceReader +public class AmoroSourceReader extends SingleThreadMultiplexSourceReaderBase< - ArcticRecordWithOffset, T, ArcticSplit, ArcticSplitState> { + AmoroRecordWithOffset, T, AmoroSplit, AmoroSplitState> { - public static final Logger LOGGER = LoggerFactory.getLogger(ArcticSourceReader.class); + public static final Logger LOGGER = LoggerFactory.getLogger(AmoroSourceReader.class); public ReaderOutput output; /** SourceEvents may be received before this#pollNext. */ private volatile boolean maxWatermarkToBeEmitted = false; - public ArcticSourceReader( + public AmoroSourceReader( ReaderFunction readerFunction, Configuration config, SourceReaderContext context, boolean populateRowTime) { super( () -> new HybridSplitReader<>(readerFunction, context), - new ArcticRecordEmitter(populateRowTime), + new AmoroRecordEmitter(populateRowTime), config, context); } @@ -80,17 +80,17 @@ public void start() { } @Override - protected void onSplitFinished(Map finishedSplitIds) { + protected void onSplitFinished(Map finishedSplitIds) { requestSplit(Lists.newArrayList(finishedSplitIds.keySet())); } @Override - protected ArcticSplitState initializedState(ArcticSplit split) { - return new ArcticSplitState(split); + protected AmoroSplitState initializedState(AmoroSplit split) { + return new AmoroSplitState(split); } @Override - protected ArcticSplit toSplitType(String splitId, ArcticSplitState splitState) { + protected AmoroSplit toSplitType(String splitId, AmoroSplitState splitState) { return splitState.toSourceSplit(); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayBatchRecords.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayBatchRecords.java index 574b392bc1..dd6d6e856c 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayBatchRecords.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayBatchRecords.java @@ -44,13 +44,13 @@ *

This class uses array to store a batch of records from the same file (with the same * fileOffset). */ -class ArrayBatchRecords implements RecordsWithSplitIds> { +class ArrayBatchRecords implements RecordsWithSplitIds> { @Nullable private String splitId; @Nullable private final Pool.Recycler recycler; @Nullable private final T[] records; private final int numberOfRecords; private final Set finishedSplits; - private final ArcticRecordWithOffset recordWithOffset; + private final AmoroRecordWithOffset recordWithOffset; // point to current read position within the records array private int position; @@ -75,7 +75,7 @@ private ArrayBatchRecords( this.numberOfRecords = numberOfRecords; this.finishedSplits = Preconditions.checkNotNull(finishedSplits, "finishedSplits can be empty but not null"); - this.recordWithOffset = new ArcticRecordWithOffset<>(); + this.recordWithOffset = new AmoroRecordWithOffset<>(); this.position = 0; } @@ -96,7 +96,7 @@ private ArrayBatchRecords( this.recordPositions = Preconditions.checkNotNull(positions, "recordPositions can't be null"); this.finishedSplits = Preconditions.checkNotNull(finishedSplits, "finishedSplits can be empty but not null"); - this.recordWithOffset = new ArcticRecordWithOffset<>(); + this.recordWithOffset = new AmoroRecordWithOffset<>(); this.position = 0; } @@ -113,7 +113,7 @@ public String nextSplit() { @Nullable @Override - public ArcticRecordWithOffset nextRecordFromSplit() { + public AmoroRecordWithOffset nextRecordFromSplit() { if (position < numberOfRecords) { setRecordWithOffset(); position++; @@ -185,7 +185,7 @@ int numberOfRecords() { * @param positions fileOffset and recordOffset for all records in this batch * @param record type */ - public static RecordsWithSplitIds> forRecords( + public static RecordsWithSplitIds> forRecords( String splitId, Pool.Recycler recycler, T[] records, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayPoolDataIteratorBatcher.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayPoolDataIteratorBatcher.java index 31599a26c4..19b2113f3b 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayPoolDataIteratorBatcher.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ArrayPoolDataIteratorBatcher.java @@ -46,7 +46,7 @@ class ArrayPoolDataIteratorBatcher implements DataIteratorBatcher { } @Override - public CloseableIterator>> batch( + public CloseableIterator>> batch( String splitId, DataIterator inputIterator) { Preconditions.checkArgument(inputIterator != null, "Input data iterator can't be null"); // lazily create pool as it is not serializable @@ -67,7 +67,7 @@ private Pool createPoolOfBatches(int numBatches) { } private class ArrayPoolBatchIterator - implements CloseableIterator>> { + implements CloseableIterator>> { private final String splitId; private final DataIterator inputIterator; @@ -85,7 +85,7 @@ public boolean hasNext() { } @Override - public RecordsWithSplitIds> next() { + public RecordsWithSplitIds> next() { if (!inputIterator.hasNext()) { throw new NoSuchElementException(); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorBatcher.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorBatcher.java index 7a07a82f2c..71f4017f09 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorBatcher.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorBatcher.java @@ -32,6 +32,6 @@ */ @FunctionalInterface public interface DataIteratorBatcher extends Serializable { - CloseableIterator>> batch( + CloseableIterator>> batch( String splitId, DataIterator inputIterator); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorReaderFunction.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorReaderFunction.java index 9cd9200ad0..d904d111b7 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorReaderFunction.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/DataIteratorReaderFunction.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read.hybrid.reader; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.flink.read.source.ChangeLogDataIterator; import org.apache.amoro.flink.read.source.DataIterator; @@ -34,11 +34,10 @@ public DataIteratorReaderFunction(DataIteratorBatcher batcher) { this.batcher = batcher; } - public abstract DataIterator createDataIterator(ArcticSplit split); + public abstract DataIterator createDataIterator(AmoroSplit split); @Override - public CloseableIterator>> apply( - ArcticSplit split) { + public CloseableIterator>> apply(AmoroSplit split) { DataIterator inputIterator = createDataIterator(split); if (inputIterator instanceof MergeOnReadDataIterator) { inputIterator.seek(0, split.asMergeOnReadSplit().recordOffset()); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/HybridSplitReader.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/HybridSplitReader.java index b7bfb50741..f079e9fbec 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/HybridSplitReader.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/HybridSplitReader.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read.hybrid.reader; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.flink.read.hybrid.split.MergeOnReadSplit; import org.apache.amoro.flink.read.hybrid.split.SnapshotSplit; @@ -41,14 +41,14 @@ /** * A hybrid source split reader that could read {@link SnapshotSplit} and {@link ChangelogSplit}. */ -public class HybridSplitReader implements SplitReader, ArcticSplit> { +public class HybridSplitReader implements SplitReader, AmoroSplit> { private static final Logger LOG = LoggerFactory.getLogger(HybridSplitReader.class); private final ReaderFunction openSplitFunction; private final int indexOfSubtask; - private final Queue splits; + private final Queue splits; - private CloseableIterator>> currentReader; + private CloseableIterator>> currentReader; private String currentSplitId; public HybridSplitReader(ReaderFunction openSplitFunction, SourceReaderContext context) { @@ -58,12 +58,12 @@ public HybridSplitReader(ReaderFunction openSplitFunction, SourceReaderContex } @Override - public RecordsWithSplitIds> fetch() throws IOException { + public RecordsWithSplitIds> fetch() throws IOException { if (currentReader == null) { if (splits.isEmpty()) { return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); } - ArcticSplit arcticSplit = splits.poll(); + AmoroSplit arcticSplit = splits.poll(); currentReader = openSplitFunction.apply(arcticSplit); currentSplitId = arcticSplit.splitId(); } @@ -81,7 +81,7 @@ public RecordsWithSplitIds> fetch() throws IOException } @Override - public void handleSplitsChanges(SplitsChange splitsChange) { + public void handleSplitsChanges(SplitsChange splitsChange) { if (!(splitsChange instanceof SplitsAddition)) { throw new UnsupportedOperationException( String.format("The SplitChange type of %s is not supported.", splitsChange.getClass())); @@ -118,7 +118,7 @@ public void close() throws Exception { } } - private RecordsWithSplitIds> finishSplit() throws IOException { + private RecordsWithSplitIds> finishSplit() throws IOException { if (currentReader != null) { currentReader.close(); currentReader = null; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderFunction.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderFunction.java index 0b570ec4cd..fe7d52e413 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderFunction.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderFunction.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read.hybrid.reader; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.iceberg.io.CloseableIterator; @@ -26,10 +26,10 @@ import java.util.function.Function; /** - * This function that accepts one {@link ArcticSplit} and produces an iterator of {@link - * ArcticRecordWithOffset}. + * This function that accepts one {@link AmoroSplit} and produces an iterator of {@link + * AmoroRecordWithOffset }. */ @FunctionalInterface public interface ReaderFunction extends Serializable, - Function>>> {} + Function>>> {} diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderStartedEvent.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderStartedEvent.java index 47216639f2..aa88bd6673 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderStartedEvent.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/ReaderStartedEvent.java @@ -20,7 +20,7 @@ import org.apache.flink.api.connector.source.SourceEvent; -/** It denotes {@link ArcticSourceReader} is starting. */ +/** It denotes {@link AmoroSourceReader} is starting. */ public class ReaderStartedEvent implements SourceEvent { private static final long serialVersionUID = 1L; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/RowDataReaderFunction.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/RowDataReaderFunction.java index a3c4db7e50..9568abda04 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/RowDataReaderFunction.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/reader/RowDataReaderFunction.java @@ -22,14 +22,14 @@ import static org.apache.amoro.utils.SchemaUtil.changeWriteSchema; import static org.apache.amoro.utils.SchemaUtil.fillUpIdentifierFields; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.source.ChangeLogDataIterator; import org.apache.amoro.flink.read.source.DataIterator; import org.apache.amoro.flink.read.source.FileScanTaskReader; -import org.apache.amoro.flink.read.source.FlinkArcticDataReader; -import org.apache.amoro.flink.read.source.FlinkArcticMORDataReader; +import org.apache.amoro.flink.read.source.FlinkAmoroDataReader; +import org.apache.amoro.flink.read.source.FlinkAmoroMORDataReader; import org.apache.amoro.flink.read.source.MergeOnReadDataIterator; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.io.AuthenticatedFileIO; import org.apache.amoro.table.PrimaryKeySpec; import org.apache.amoro.utils.NodeFilter; @@ -43,7 +43,7 @@ import java.util.Collections; /** - * This Function accept a {@link ArcticSplit} and produces an {@link DataIterator} of {@link + * This Function accept a {@link AmoroSplit} and produces an {@link DataIterator} of {@link * RowData}. */ public class RowDataReaderFunction extends DataIteratorReaderFunction { @@ -111,10 +111,10 @@ public RowDataReaderFunction( } @Override - public DataIterator createDataIterator(ArcticSplit split) { + public DataIterator createDataIterator(AmoroSplit split) { if (split.isMergeOnReadSplit()) { - FlinkArcticMORDataReader morDataReader = - new FlinkArcticMORDataReader( + FlinkAmoroMORDataReader morDataReader = + new FlinkAmoroMORDataReader( io, tableSchema, readSchema, @@ -127,7 +127,7 @@ public DataIterator createDataIterator(ArcticSplit split) { morDataReader, split.asMergeOnReadSplit().keyedTableScanTask(), io); } else if (split.isSnapshotSplit()) { FileScanTaskReader rowDataReader = - new FlinkArcticDataReader( + new FlinkAmoroDataReader( io, tableSchema, readSchema, @@ -144,7 +144,7 @@ public DataIterator createDataIterator(ArcticSplit split) { this::removeArcticMetaColumn); } else if (split.isChangelogSplit()) { FileScanTaskReader rowDataReader = - new FlinkArcticDataReader( + new FlinkAmoroDataReader( io, wrapArcticFileOffsetColumnMeta(tableSchema), wrapArcticFileOffsetColumnMeta(readSchema), @@ -178,10 +178,10 @@ long arcticFileOffset(RowData rowData) { /** * @param rowData It may have more columns than readSchema. Refer to {@link - * FlinkArcticDataReader}'s annotation. + * FlinkAmoroDataReader}'s annotation. */ RowData removeArcticMetaColumn(RowData rowData) { - return ArcticUtils.removeArcticMetaColumn(rowData, columnSize); + return AmoroUtils.removeArcticMetaColumn(rowData, columnSize); } RowData transformRowKind(ChangeLogDataIterator.ChangeActionTrans trans) { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplit.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplit.java similarity index 93% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplit.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplit.java index d085d60982..042e248b11 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplit.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplit.java @@ -24,7 +24,7 @@ import java.io.Serializable; /** An abstract arctic source split. */ -public abstract class ArcticSplit implements SourceSplit, Serializable, Comparable { +public abstract class AmoroSplit implements SourceSplit, Serializable, Comparable { private static final long serialVersionUID = 1L; public abstract Integer taskIndex(); @@ -76,9 +76,9 @@ public final MergeOnReadSplit asMergeOnReadSplit() { public abstract void updateOffset(Object[] recordOffsets); @Override - public int compareTo(ArcticSplit that) { + public int compareTo(AmoroSplit that) { return this.taskIndex().compareTo(that.taskIndex()); } - public abstract ArcticSplit copy(); + public abstract AmoroSplit copy(); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplitSerializer.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplitSerializer.java similarity index 90% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplitSerializer.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplitSerializer.java index 8b1c57b9e6..42235e74ce 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplitSerializer.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplitSerializer.java @@ -25,9 +25,9 @@ import java.io.IOException; -/** Serializer that serializes and deserializes {@link ArcticSplit}. */ -public class ArcticSplitSerializer implements SimpleVersionedSerializer { - public static final ArcticSplitSerializer INSTANCE = new ArcticSplitSerializer(); +/** Serializer that serializes and deserializes {@link AmoroSplit}. */ +public class AmoroSplitSerializer implements SimpleVersionedSerializer { + public static final AmoroSplitSerializer INSTANCE = new AmoroSplitSerializer(); private static final int VERSION = 1; private static final byte SNAPSHOT_SPLIT_FLAG = 1; @@ -40,7 +40,7 @@ public int getVersion() { } @Override - public byte[] serialize(ArcticSplit split) throws IOException { + public byte[] serialize(AmoroSplit split) throws IOException { if (split == null) { return new byte[0]; } @@ -64,7 +64,7 @@ public byte[] serialize(ArcticSplit split) throws IOException { } @Override - public ArcticSplit deserialize(int version, byte[] serialized) throws IOException { + public AmoroSplit deserialize(int version, byte[] serialized) throws IOException { if (serialized.length == 0) { return null; } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplitState.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplitState.java similarity index 94% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplitState.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplitState.java index 7495286ad9..7dd2b00222 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ArcticSplitState.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/AmoroSplitState.java @@ -21,19 +21,19 @@ import org.apache.flink.util.FlinkRuntimeException; /** This is the mutable state for per arctic source split. */ -public class ArcticSplitState { - private final ArcticSplit arcticSplit; +public class AmoroSplitState { + private final AmoroSplit arcticSplit; private int currentInsertFileOffset; private long currentInsertRecordOffset; private int currentDeleteFileOffset; private long currentDeleteRecordOffset; - public ArcticSplitState(ArcticSplit arcticSplit) { + public AmoroSplitState(AmoroSplit arcticSplit) { this.arcticSplit = arcticSplit; } - public ArcticSplit toSourceSplit() { + public AmoroSplit toSourceSplit() { if (arcticSplit.isMergeOnReadSplit()) { MergeOnReadSplit mergeOnReadSplit = (MergeOnReadSplit) arcticSplit; mergeOnReadSplit.updateOffset( diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ChangelogSplit.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ChangelogSplit.java index a549c5f8bc..58cefef5d5 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ChangelogSplit.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/ChangelogSplit.java @@ -29,7 +29,7 @@ import java.util.Optional; /** A changelog split generated during planning change table. */ -public class ChangelogSplit extends ArcticSplit { +public class ChangelogSplit extends AmoroSplit { private static final long serialVersionUID = 1L; private final int taskIndex; private final Collection insertScanTasks; @@ -80,7 +80,7 @@ public void updateOffset(Object[] offsets) { } @Override - public ArcticSplit copy() { + public AmoroSplit copy() { return new ChangelogSplit(insertScanTasks, deleteScanTasks, taskIndex); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/MergeOnReadSplit.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/MergeOnReadSplit.java index 68bf4c3618..6214eb281b 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/MergeOnReadSplit.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/MergeOnReadSplit.java @@ -23,7 +23,7 @@ import org.apache.flink.util.Preconditions; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; -public class MergeOnReadSplit extends ArcticSplit { +public class MergeOnReadSplit extends AmoroSplit { private static final long serialVersionUID = 1L; private final int taskIndex; private final KeyedTableScanTask keyedTableScanTask; @@ -51,7 +51,7 @@ public void updateOffset(Object[] offsets) { } @Override - public ArcticSplit copy() { + public AmoroSplit copy() { return new MergeOnReadSplit(taskIndex, keyedTableScanTask); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/SnapshotSplit.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/SnapshotSplit.java index c49b396b78..e364ac4f54 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/SnapshotSplit.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/SnapshotSplit.java @@ -28,7 +28,7 @@ import java.util.Collection; /** A snapshot split generated during planning base table. */ -public class SnapshotSplit extends ArcticSplit { +public class SnapshotSplit extends AmoroSplit { private static final long serialVersionUID = 1L; private final int taskIndex; private final Collection insertScanTasks; @@ -79,7 +79,7 @@ public void updateOffset(Object[] offsets) { } @Override - public ArcticSplit copy() { + public AmoroSplit copy() { return new SnapshotSplit(insertScanTasks, taskIndex); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/TemporalJoinSplits.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/TemporalJoinSplits.java index a03d5ceb31..4b8f8ea543 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/TemporalJoinSplits.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/hybrid/split/TemporalJoinSplits.java @@ -51,7 +51,7 @@ public class TemporalJoinSplits implements Serializable { /** transient because it is necessary to notify reader again after failover. */ private transient boolean hasNotifiedReader = false; - public TemporalJoinSplits(Collection splits, MetricGroup metricGroup) { + public TemporalJoinSplits(Collection splits, MetricGroup metricGroup) { Preconditions.checkNotNull(splits, "plan splits should not be null"); this.splits = splits.stream().map(SourceSplit::splitId).collect(Collectors.toMap((k) -> k, (i) -> false)); @@ -68,7 +68,7 @@ public Map getSplits() { return splits; } - public synchronized void addSplitsBack(Collection splits) { + public synchronized void addSplitsBack(Collection splits) { if (this.splits == null || CollectionUtil.isNullOrEmpty(splits)) { return; } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/ArcticScanContext.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/AmoroScanContext.java similarity index 97% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/ArcticScanContext.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/AmoroScanContext.java index 1a24afdfca..e744621f5d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/ArcticScanContext.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/AmoroScanContext.java @@ -37,15 +37,15 @@ import java.util.Map; import java.util.Objects; -/** This is an arctic source scan context. */ -public class ArcticScanContext extends ScanContext implements Serializable { +/** This is an amoro source scan context. */ +public class AmoroScanContext extends ScanContext implements Serializable { private static final long serialVersionUID = 1L; private final String scanStartupMode; private final boolean batchMode; - protected ArcticScanContext(Builder builder) { + protected AmoroScanContext(Builder builder) { super( builder.caseSensitive, builder.snapshotId, @@ -348,7 +348,7 @@ public Builder fromProperties(Map properties) { .maxAllowedPlanningFailures(maxAllowedPlanningFailures); } - public ArcticScanContext build() { + public AmoroScanContext build() { scanStartupMode = scanStartupMode == null ? null : scanStartupMode.toLowerCase(); Preconditions.checkArgument( Objects.isNull(scanStartupMode) @@ -364,7 +364,7 @@ public ArcticScanContext build() { !(isStreaming && batchMode), String.format( "only support %s = false when execution.runtime-mode is batch", STREAMING.key())); - return new ArcticScanContext(this); + return new AmoroScanContext(this); } } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkArcticDataReader.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkAmoroDataReader.java similarity index 96% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkArcticDataReader.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkAmoroDataReader.java index 6b84514ef8..22447bf722 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkArcticDataReader.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkAmoroDataReader.java @@ -53,11 +53,11 @@ * DeleteFilter}. It shall be projected before sent to downstream. This can be processed in {@link * DataIterator#next()} */ -public class FlinkArcticDataReader extends AbstractAdaptHiveUnkeyedDataReader +public class FlinkAmoroDataReader extends AbstractAdaptHiveUnkeyedDataReader implements FileScanTaskReader { private static final long serialVersionUID = -6773693031945244386L; - public FlinkArcticDataReader( + public FlinkAmoroDataReader( AuthenticatedFileIO fileIO, Schema tableSchema, Schema projectedSchema, @@ -75,7 +75,7 @@ public FlinkArcticDataReader( reuseContainer); } - public FlinkArcticDataReader( + public FlinkAmoroDataReader( AuthenticatedFileIO fileIO, Schema tableSchema, Schema projectedSchema, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkArcticMORDataReader.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkAmoroMORDataReader.java similarity index 96% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkArcticMORDataReader.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkAmoroMORDataReader.java index a7355ced88..3dc3ae0f6c 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkArcticMORDataReader.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/FlinkAmoroMORDataReader.java @@ -39,8 +39,8 @@ import java.util.function.BiFunction; import java.util.function.Function; -public class FlinkArcticMORDataReader extends AbstractAdaptHiveKeyedDataReader { - public FlinkArcticMORDataReader( +public class FlinkAmoroMORDataReader extends AbstractAdaptHiveKeyedDataReader { + public FlinkAmoroMORDataReader( AuthenticatedFileIO fileIO, Schema tableSchema, Schema projectedSchema, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/MergeOnReadDataIterator.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/MergeOnReadDataIterator.java index 92d8a8197a..4cba0772e3 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/MergeOnReadDataIterator.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/read/source/MergeOnReadDataIterator.java @@ -37,12 +37,12 @@ public class MergeOnReadDataIterator extends DataIterator { private final CloseableIterator iterator; public MergeOnReadDataIterator( - FlinkArcticMORDataReader flinkArcticMORDataReader, + FlinkAmoroMORDataReader flinkAmoroMORDataReader, KeyedTableScanTask keyedTableScanTask, AuthenticatedFileIO io) { super(); this.iterator = - IteratorWithIO.of(io, io.doAs(() -> flinkArcticMORDataReader.readData(keyedTableScanTask))); + IteratorWithIO.of(io, io.doAs(() -> flinkAmoroMORDataReader.readData(keyedTableScanTask))); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticDynamicSink.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroDynamicSink.java similarity index 87% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticDynamicSink.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroDynamicSink.java index 1f5cb4bda7..fb9c0cb33b 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticDynamicSink.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroDynamicSink.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.table; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.write.FlinkSink; import org.apache.amoro.table.MixedTable; import org.apache.flink.streaming.api.datastream.DataStream; @@ -39,18 +39,17 @@ import java.util.Map; /** Flink table api that generates sink operators. */ -public class ArcticDynamicSink - implements DynamicTableSink, SupportsPartitioning, SupportsOverwrite { +public class AmoroDynamicSink implements DynamicTableSink, SupportsPartitioning, SupportsOverwrite { - public static final Logger LOG = LoggerFactory.getLogger(ArcticDynamicSink.class); + public static final Logger LOG = LoggerFactory.getLogger(AmoroDynamicSink.class); - private final ArcticTableLoader tableLoader; + private final AmoroTableLoader tableLoader; private final CatalogTable flinkTable; private final boolean primaryKeyExisted; private boolean overwrite = false; - ArcticDynamicSink( - CatalogTable flinkTable, ArcticTableLoader tableLoader, boolean primaryKeyExisted) { + AmoroDynamicSink( + CatalogTable flinkTable, AmoroTableLoader tableLoader, boolean primaryKeyExisted) { this.tableLoader = tableLoader; this.flinkTable = flinkTable; this.primaryKeyExisted = primaryKeyExisted; @@ -70,7 +69,7 @@ public ChangelogMode getChangelogMode(ChangelogMode changelogMode) { @Override public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { - MixedTable table = ArcticUtils.loadArcticTable(tableLoader); + MixedTable table = AmoroUtils.loadArcticTable(tableLoader); return new DataStreamSinkProvider() { @Override @@ -98,7 +97,7 @@ public DynamicTableSink copy() { @Override public String asSummaryString() { - return "arctic"; + return "amoro"; } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticDynamicSource.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroDynamicSource.java similarity index 96% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticDynamicSource.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroDynamicSource.java index f69d4c34ed..a5293896c8 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticDynamicSource.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroDynamicSource.java @@ -18,13 +18,13 @@ package org.apache.amoro.flink.table; -import org.apache.amoro.flink.lookup.ArcticRowDataLookupFunction; +import org.apache.amoro.flink.lookup.AmoroRowDataLookupFunction; import org.apache.amoro.flink.lookup.KVTableFactory; import org.apache.amoro.flink.lookup.filter.RowDataPredicate; import org.apache.amoro.flink.lookup.filter.RowDataPredicateExpressionVisitor; import org.apache.amoro.flink.read.hybrid.reader.DataIteratorReaderFunction; import org.apache.amoro.flink.read.hybrid.reader.RowDataReaderFunction; -import org.apache.amoro.flink.read.source.FlinkArcticMORDataReader; +import org.apache.amoro.flink.read.source.FlinkAmoroMORDataReader; import org.apache.amoro.flink.util.FilterUtil; import org.apache.amoro.flink.util.IcebergAndFlinkFilters; import org.apache.amoro.hive.io.reader.AbstractAdaptHiveKeyedDataReader; @@ -74,7 +74,7 @@ import java.util.stream.Collectors; /** Flink table api that generates source operators. */ -public class ArcticDynamicSource +public class AmoroDynamicSource implements ScanTableSource, SupportsFilterPushDown, SupportsProjectionPushDown, @@ -82,7 +82,7 @@ public class ArcticDynamicSource SupportsWatermarkPushDown, LookupTableSource { - private static final Logger LOG = LoggerFactory.getLogger(ArcticDynamicSource.class); + private static final Logger LOG = LoggerFactory.getLogger(AmoroDynamicSource.class); protected final String tableName; @@ -93,7 +93,7 @@ public class ArcticDynamicSource protected int[] projectFields; protected List filters; protected ResolvedExpression flinkExpression; - protected final ArcticTableLoader tableLoader; + protected final AmoroTableLoader tableLoader; @Nullable protected WatermarkStrategy watermarkStrategy; @@ -104,12 +104,12 @@ public class ArcticDynamicSource * @param properties With all ArcticTable properties and sql options * @param tableLoader */ - public ArcticDynamicSource( + public AmoroDynamicSource( String tableName, ScanTableSource arcticDynamicSource, MixedTable mixedTable, Map properties, - ArcticTableLoader tableLoader) { + AmoroTableLoader tableLoader) { this.tableName = tableName; this.arcticDynamicSource = arcticDynamicSource; this.mixedTable = mixedTable; @@ -117,12 +117,12 @@ public ArcticDynamicSource( this.tableLoader = tableLoader; } - public ArcticDynamicSource( + public AmoroDynamicSource( String tableName, ScanTableSource arcticDynamicSource, MixedTable mixedTable, Map properties, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, int[] projectFields, List filters, ResolvedExpression flinkExpression) { @@ -153,7 +153,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { @Override public DynamicTableSource copy() { - return new ArcticDynamicSource( + return new AmoroDynamicSource( tableName, arcticDynamicSource, mixedTable, @@ -266,7 +266,7 @@ protected LookupFunction getLookupFunction(int[] joinKeys) { DataIteratorReaderFunction readerFunction = generateReaderFunction(mixedTable, projectedSchema); - return new ArcticRowDataLookupFunction( + return new AmoroRowDataLookupFunction( KVTableFactory.INSTANCE, mixedTable, joinKeyNames, @@ -296,7 +296,7 @@ protected AbstractAdaptHiveKeyedDataReader generateMORReader( MixedTable mixedTable, Schema projectedSchema) { BiFunction convertConstant = new ConvertTask(); - return new FlinkArcticMORDataReader( + return new FlinkAmoroMORDataReader( mixedTable.io(), mixedTable.schema(), projectedSchema, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticFileSource.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroFileSource.java similarity index 95% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticFileSource.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroFileSource.java index 1a308e8e99..35133659cd 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticFileSource.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroFileSource.java @@ -57,14 +57,14 @@ import java.util.List; /** Flink table api that generates arctic base/change file source operators. */ -public class ArcticFileSource +public class AmoroFileSource implements ScanTableSource, SupportsFilterPushDown, SupportsProjectionPushDown, SupportsLimitPushDown, SupportsWatermarkPushDown { - private static final Logger LOG = LoggerFactory.getLogger(ArcticFileSource.class); + private static final Logger LOG = LoggerFactory.getLogger(AmoroFileSource.class); private int[] projectedFields; private long limit; @@ -72,12 +72,12 @@ public class ArcticFileSource private final MixedTable table; @Nullable protected WatermarkStrategy watermarkStrategy; - private final ArcticTableLoader loader; + private final AmoroTableLoader loader; private final TableSchema tableSchema; private final ReadableConfig readableConfig; private final boolean batchMode; - private ArcticFileSource(ArcticFileSource toCopy) { + private AmoroFileSource(AmoroFileSource toCopy) { this.loader = toCopy.loader; this.tableSchema = toCopy.tableSchema; this.projectedFields = toCopy.projectedFields; @@ -89,8 +89,8 @@ private ArcticFileSource(ArcticFileSource toCopy) { this.batchMode = toCopy.batchMode; } - public ArcticFileSource( - ArcticTableLoader loader, + public AmoroFileSource( + AmoroTableLoader loader, TableSchema tableSchema, int[] projectedFields, MixedTable table, @@ -108,8 +108,8 @@ public ArcticFileSource( this.batchMode = batchMode; } - public ArcticFileSource( - ArcticTableLoader loader, + public AmoroFileSource( + AmoroTableLoader loader, TableSchema tableSchema, MixedTable table, ReadableConfig readableConfig, @@ -224,7 +224,7 @@ public boolean isBounded() { @Override public DynamicTableSource copy() { - return new ArcticFileSource(this); + return new AmoroFileSource(this); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticTableLoader.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroTableLoader.java similarity index 91% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticTableLoader.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroTableLoader.java index 0febdbf06e..7f41f46a6d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/ArcticTableLoader.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/AmoroTableLoader.java @@ -33,7 +33,7 @@ import java.util.Map; /** load a proxy table contains both arctic table properties and flink table properties */ -public class ArcticTableLoader implements TableLoader { +public class AmoroTableLoader implements TableLoader { private static final long serialVersionUID = 1L; @@ -48,45 +48,45 @@ public class ArcticTableLoader implements TableLoader { protected transient MixedFormatCatalog arcticCatalog; - public static ArcticTableLoader of( + public static AmoroTableLoader of( TableIdentifier tableIdentifier, InternalCatalogBuilder catalogBuilder) { return of(tableIdentifier, catalogBuilder, new HashMap<>()); } - public static ArcticTableLoader of( + public static AmoroTableLoader of( TableIdentifier tableIdentifier, InternalCatalogBuilder catalogBuilder, Map flinkTableProperties) { - return new ArcticTableLoader(tableIdentifier, catalogBuilder, flinkTableProperties); + return new AmoroTableLoader(tableIdentifier, catalogBuilder, flinkTableProperties); } - public static ArcticTableLoader of( + public static AmoroTableLoader of( TableIdentifier tableIdentifier, Map flinkTableProperties) { String metastoreUrl = flinkTableProperties.get(CatalogFactoryOptions.METASTORE_URL.key()); - return new ArcticTableLoader( + return new AmoroTableLoader( tableIdentifier, InternalCatalogBuilder.builder().metastoreUrl(metastoreUrl), flinkTableProperties); } - public static ArcticTableLoader of( + public static AmoroTableLoader of( TableIdentifier tableIdentifier, String metastoreUrl, Map flinkTableProperties) { - return new ArcticTableLoader( + return new AmoroTableLoader( tableIdentifier, InternalCatalogBuilder.builder().metastoreUrl(metastoreUrl), flinkTableProperties); } - protected ArcticTableLoader( + protected AmoroTableLoader( TableIdentifier tableIdentifier, InternalCatalogBuilder catalogBuilder, Map flinkTableProperties) { this(tableIdentifier, catalogBuilder, flinkTableProperties, null); } - protected ArcticTableLoader( + protected AmoroTableLoader( TableIdentifier tableIdentifier, InternalCatalogBuilder catalogBuilder, Map flinkTableProperties, @@ -138,7 +138,7 @@ public Table loadTable() { @Override public TableLoader clone() { - return new ArcticTableLoader( + return new AmoroTableLoader( tableIdentifier, catalogBuilder, flinkTableProperties, loadBaseForKeyedTable); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/DynamicTableFactory.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/DynamicTableFactory.java index c706a41718..e946751803 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/DynamicTableFactory.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/DynamicTableFactory.java @@ -34,7 +34,7 @@ import org.apache.amoro.flink.catalog.MixedCatalog; import org.apache.amoro.flink.catalog.factories.CatalogFactoryOptions; import org.apache.amoro.flink.table.descriptors.ArcticValidator; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.util.CompatibleFlinkPropertyUtil; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; @@ -67,7 +67,7 @@ import java.util.Properties; import java.util.Set; -/** A factory generates {@link ArcticDynamicSource} and {@link ArcticDynamicSink} */ +/** A factory generates {@link AmoroDynamicSource} and {@link AmoroDynamicSink} */ public class DynamicTableFactory implements DynamicTableSourceFactory, DynamicTableSinkFactory { private static final Logger LOG = LoggerFactory.getLogger(DynamicTableFactory.class); public static final String IDENTIFIER = "arctic"; @@ -115,9 +115,9 @@ public DynamicTableSource createDynamicTableSource(Context context) { } else { objectPath = new ObjectPath(identifier.getDatabaseName(), identifier.getObjectName()); } - ArcticTableLoader tableLoader = + AmoroTableLoader tableLoader = createTableLoader(objectPath, actualCatalogName, actualBuilder, options.toMap()); - MixedTable mixedTable = ArcticUtils.loadArcticTable(tableLoader); + MixedTable mixedTable = AmoroUtils.loadArcticTable(tableLoader); Configuration confWithAll = Configuration.fromMap(mixedTable.properties()); @@ -150,7 +150,7 @@ public DynamicTableSource createDynamicTableSource(Context context) { boolean batchMode = context.getConfiguration().get(RUNTIME_MODE).equals(BATCH); LOG.info("Building a file reader in {} runtime mode", batchMode ? "batch" : "streaming"); arcticDynamicSource = - new ArcticFileSource(tableLoader, tableSchema, mixedTable, confWithAll, batchMode); + new AmoroFileSource(tableLoader, tableSchema, mixedTable, confWithAll, batchMode); break; case ArcticValidator.ARCTIC_READ_LOG: default: @@ -169,30 +169,30 @@ protected DynamicTableSource generateDynamicTableSource( String tableName, ScanTableSource arcticDynamicSource, MixedTable mixedTable, - ArcticTableLoader tableLoader) { - return new ArcticDynamicSource( + AmoroTableLoader tableLoader) { + return new AmoroDynamicSource( tableName, arcticDynamicSource, mixedTable, mixedTable.properties(), tableLoader); } @Override - public ArcticDynamicSink createDynamicTableSink(Context context) { + public AmoroDynamicSink createDynamicTableSink(Context context) { CatalogTable catalogTable = context.getCatalogTable(); ObjectIdentifier identifier = context.getObjectIdentifier(); Map options = catalogTable.getOptions(); - ArcticTableLoader tableLoader = + AmoroTableLoader tableLoader = createTableLoader( new ObjectPath(identifier.getDatabaseName(), identifier.getObjectName()), internalCatalogName, internalCatalogBuilder, options); - MixedTable table = ArcticUtils.loadArcticTable(tableLoader); - return new ArcticDynamicSink(catalogTable, tableLoader, table.isKeyedTable()); + MixedTable table = AmoroUtils.loadArcticTable(tableLoader); + return new AmoroDynamicSink(catalogTable, tableLoader, table.isKeyedTable()); } - private static ArcticTableLoader createTableLoader( + private static AmoroTableLoader createTableLoader( ObjectPath tablePath, String internalCatalogName, InternalCatalogBuilder catalogBuilder, @@ -201,7 +201,7 @@ private static ArcticTableLoader createTableLoader( TableIdentifier.of( internalCatalogName, tablePath.getDatabaseName(), tablePath.getObjectName()); - return ArcticTableLoader.of(identifier, catalogBuilder, flinkTableProperties); + return AmoroTableLoader.of(identifier, catalogBuilder, flinkTableProperties); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/FlinkSource.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/FlinkSource.java index 55372137ec..7b30063c1b 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/FlinkSource.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/table/FlinkSource.java @@ -19,11 +19,11 @@ package org.apache.amoro.flink.table; import org.apache.amoro.flink.interceptor.ProxyFactory; -import org.apache.amoro.flink.read.ArcticSource; +import org.apache.amoro.flink.read.AmoroSource; import org.apache.amoro.flink.read.hybrid.reader.RowDataReaderFunction; -import org.apache.amoro.flink.read.source.ArcticScanContext; +import org.apache.amoro.flink.read.source.AmoroScanContext; import org.apache.amoro.flink.table.descriptors.ArcticValidator; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.util.CompatibleFlinkPropertyUtil; import org.apache.amoro.flink.util.IcebergClassUtil; import org.apache.amoro.flink.util.ProxyUtil; @@ -70,14 +70,14 @@ public static final class Builder { private ProviderContext context; private StreamExecutionEnvironment env; private MixedTable mixedTable; - private ArcticTableLoader tableLoader; + private AmoroTableLoader tableLoader; private TableSchema projectedSchema; private List filters; private ReadableConfig flinkConf = new Configuration(); private final Map properties = new HashMap<>(); private long limit = -1L; private WatermarkStrategy watermarkStrategy = WatermarkStrategy.noWatermarks(); - private final ArcticScanContext.Builder contextBuilder = ArcticScanContext.arcticBuilder(); + private final AmoroScanContext.Builder contextBuilder = AmoroScanContext.arcticBuilder(); private boolean batchMode = false; private Builder() {} @@ -98,7 +98,7 @@ public Builder arcticTable(MixedTable mixedTable) { return this; } - public Builder tableLoader(ArcticTableLoader tableLoader) { + public Builder tableLoader(AmoroTableLoader tableLoader) { this.tableLoader = tableLoader; return this; } @@ -174,7 +174,7 @@ public DataStream build() { org.apache.amoro.flink.FlinkSchemaUtil.filterWatermark(projectedSchema)); } } - ArcticScanContext scanContext = + AmoroScanContext scanContext = contextBuilder.fromProperties(properties).batchMode(batchMode).build(); RowDataReaderFunction rowDataReaderFunction = @@ -191,7 +191,7 @@ public DataStream build() { flinkConf.getOptional(ArcticValidator.SCAN_PARALLELISM).orElse(env.getParallelism()); DataStreamSource sourceStream = env.fromSource( - new ArcticSource<>( + new AmoroSource<>( tableLoader, scanContext, rowDataReaderFunction, @@ -199,7 +199,7 @@ public DataStream build() { mixedTable.name(), dimTable), watermarkStrategy, - ArcticSource.class.getName()) + AmoroSource.class.getName()) .setParallelism(scanParallelism); context.generateUid(ARCTIC_FILE_TRANSFORMATION).ifPresent(sourceStream::uid); return sourceStream; @@ -209,7 +209,7 @@ private void loadTableIfNeeded() { if (tableLoader == null || mixedTable != null) { return; } - mixedTable = ArcticUtils.loadArcticTable(tableLoader); + mixedTable = AmoroUtils.loadArcticTable(tableLoader); properties.putAll(mixedTable.properties()); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/ArcticUtils.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/AmoroUtils.java similarity index 96% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/ArcticUtils.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/AmoroUtils.java index 51a80fe23c..aa2177941c 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/ArcticUtils.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/util/AmoroUtils.java @@ -31,9 +31,9 @@ import org.apache.amoro.flink.metric.MetricsGenerator; import org.apache.amoro.flink.shuffle.LogRecordV1; import org.apache.amoro.flink.shuffle.ShuffleHelper; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.table.descriptors.ArcticValidator; -import org.apache.amoro.flink.write.ArcticLogWriter; +import org.apache.amoro.flink.write.AmoroLogWriter; import org.apache.amoro.flink.write.AutomaticLogWriter; import org.apache.amoro.flink.write.hidden.HiddenLogWriter; import org.apache.amoro.flink.write.hidden.kafka.HiddenKafkaFactory; @@ -65,11 +65,11 @@ import java.util.stream.Collectors; /** An util that loads arctic table, build arctic log writer and so on. */ -public class ArcticUtils { +public class AmoroUtils { - public static final Logger LOG = LoggerFactory.getLogger(ArcticUtils.class); + public static final Logger LOG = LoggerFactory.getLogger(AmoroUtils.class); - public static MixedTable loadArcticTable(ArcticTableLoader tableLoader) { + public static MixedTable loadArcticTable(AmoroTableLoader tableLoader) { tableLoader.open(); MixedTable table = tableLoader.loadArcticTable(); try { @@ -143,14 +143,14 @@ public static boolean arcticWALWriterEnable( * @param watermarkWriteGap watermark gap that triggers automatic writing to log storage * @return ArcticLogWriter */ - public static ArcticLogWriter buildArcticLogWriter( + public static AmoroLogWriter buildArcticLogWriter( Map properties, @Nullable Properties producerConfig, @Nullable String topic, TableSchema tableSchema, String arcticEmitMode, ShuffleHelper helper, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, Duration watermarkWriteGap) { if (!arcticWALWriterEnable(properties, arcticEmitMode)) { return null; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticFileWriter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroFileWriter.java similarity index 91% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticFileWriter.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroFileWriter.java index 115cf3dd3d..c84b346461 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticFileWriter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroFileWriter.java @@ -21,8 +21,8 @@ import org.apache.amoro.data.DataTreeNode; import org.apache.amoro.flink.shuffle.ShuffleKey; import org.apache.amoro.flink.shuffle.ShuffleRulePolicy; -import org.apache.amoro.flink.table.ArcticTableLoader; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.table.AmoroTableLoader; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.table.MixedTable; import org.apache.commons.lang.ArrayUtils; import org.apache.flink.annotation.VisibleForTesting; @@ -45,18 +45,18 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -/** This is arctic table includes writing file data to un keyed table and keyed table. */ -public class ArcticFileWriter extends AbstractStreamOperator +/** This is amoro table includes writing file data to un keyed table and keyed table. */ +public class AmoroFileWriter extends AbstractStreamOperator implements OneInputStreamOperator, BoundedOneInput { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(ArcticFileWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(AmoroFileWriter.class); private final ShuffleRulePolicy shuffleRule; private final TaskWriterFactory taskWriterFactory; private final int minFileSplitCount; - private final ArcticTableLoader tableLoader; + private final AmoroTableLoader tableLoader; private final boolean submitEmptySnapshot; private transient TaskWriter writer; @@ -70,11 +70,11 @@ public class ArcticFileWriter extends AbstractStreamOperator */ private transient MixedTable table; - public ArcticFileWriter( + public AmoroFileWriter( ShuffleRulePolicy shuffleRule, TaskWriterFactory taskWriterFactory, int minFileSplitCount, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, boolean upsert, boolean submitEmptySnapshot) { this.shuffleRule = shuffleRule; @@ -92,7 +92,7 @@ public ArcticFileWriter( @Override public void open() { this.attemptId = getRuntimeContext().getAttemptNumber(); - table = ArcticUtils.loadArcticTable(tableLoader); + table = AmoroUtils.loadArcticTable(tableLoader); long mask = getMask(subTaskId); initTaskWriterFactory(mask); @@ -113,8 +113,8 @@ public void snapshotState(StateSnapshotContext context) throws Exception { } private void initTaskWriterFactory(long mask) { - if (taskWriterFactory instanceof ArcticRowDataTaskWriterFactory) { - ((ArcticRowDataTaskWriterFactory) taskWriterFactory).setMask(mask); + if (taskWriterFactory instanceof AmoroRowDataTaskWriterFactory) { + ((AmoroRowDataTaskWriterFactory) taskWriterFactory).setMask(mask); } taskWriterFactory.initialize(subTaskId, attemptId); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticLogWriter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroLogWriter.java similarity index 90% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticLogWriter.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroLogWriter.java index 225b28672e..13e7dec1a7 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticLogWriter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroLogWriter.java @@ -23,6 +23,6 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.table.data.RowData; -/** This is a common abstract arctic log writer. */ -public abstract class ArcticLogWriter extends AbstractStreamOperator +/** This is a common abstract amoro log writer. */ +public abstract class AmoroLogWriter extends AbstractStreamOperator implements OneInputStreamOperator, BoundedOneInput {} diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticRowDataTaskWriterFactory.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroRowDataTaskWriterFactory.java similarity index 91% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticRowDataTaskWriterFactory.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroRowDataTaskWriterFactory.java index 4815564b55..5193928c70 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/ArcticRowDataTaskWriterFactory.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AmoroRowDataTaskWriterFactory.java @@ -26,8 +26,8 @@ import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -/** This is an Arctic table writer factory. */ -public class ArcticRowDataTaskWriterFactory implements TaskWriterFactory { +/** This is an amoro table writer factory. */ +public class AmoroRowDataTaskWriterFactory implements TaskWriterFactory { private final MixedTable table; private final RowType flinkSchema; @@ -39,7 +39,7 @@ public class ArcticRowDataTaskWriterFactory implements TaskWriterFactory */ -public class ArcticWriter extends AbstractStreamOperator +public class AmoroWriter extends AbstractStreamOperator implements OneInputStreamOperator, BoundedOneInput { private transient Meter meterFlowRate; @@ -53,13 +53,13 @@ public class ArcticWriter extends AbstractStreamOperator private transient Meter meterSpeed; private final AbstractStreamOperator fileWriter; - private final ArcticLogWriter logWriter; + private final AmoroLogWriter logWriter; private final MetricsGenerator metricsGenerator; private static final String INFLUXDB_TAG_NAME = "arctic_task_id"; - public ArcticWriter( - ArcticLogWriter logWriter, + public AmoroWriter( + AmoroLogWriter logWriter, AbstractStreamOperator fileWriter, MetricsGenerator metricsGenerator) { this.logWriter = logWriter; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticDoubleWriteStatus.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticDoubleWriteStatus.java index ae2fd235c6..a41d5067ed 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticDoubleWriteStatus.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticDoubleWriteStatus.java @@ -18,9 +18,9 @@ package org.apache.amoro.flink.write; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.table.descriptors.ArcticValidator; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.table.MixedTable; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.iceberg.UpdateProperties; @@ -36,14 +36,14 @@ public class AutomaticDoubleWriteStatus implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(AutomaticDoubleWriteStatus.class); private static final long serialVersionUID = 1L; - private final ArcticTableLoader tableLoader; + private final AmoroTableLoader tableLoader; private final AutomaticWriteSpecification specification; private MixedTable table; private transient boolean shouldDoubleWrite = false; private int subtaskId; public AutomaticDoubleWriteStatus( - ArcticTableLoader tableLoader, Duration writeLogstoreWatermarkGap) { + AmoroTableLoader tableLoader, Duration writeLogstoreWatermarkGap) { this.tableLoader = tableLoader; this.specification = new AutomaticWriteSpecification(writeLogstoreWatermarkGap); } @@ -53,7 +53,7 @@ public void setup(int indexOfThisSubtask) { } public void open() { - table = ArcticUtils.loadArcticTable(tableLoader); + table = AmoroUtils.loadArcticTable(tableLoader); sync(); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticLogWriter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticLogWriter.java index 50c2cccadf..6e752fc59d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticLogWriter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/AutomaticLogWriter.java @@ -19,7 +19,7 @@ package org.apache.amoro.flink.write; import org.apache.amoro.flink.shuffle.ShuffleHelper; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.table.descriptors.ArcticValidator; import org.apache.amoro.flink.write.hidden.HiddenLogWriter; import org.apache.amoro.flink.write.hidden.LogMsgFactory; @@ -42,9 +42,9 @@ * timestamp is greater than the watermark of all subtasks plus the {@link * ArcticValidator#AUTO_EMIT_LOGSTORE_WATERMARK_GAP} value. */ -public class AutomaticLogWriter extends ArcticLogWriter { +public class AutomaticLogWriter extends AmoroLogWriter { private final AutomaticDoubleWriteStatus status; - private final ArcticLogWriter arcticLogWriter; + private final AmoroLogWriter amoroLogWriter; public AutomaticLogWriter( Schema schema, @@ -54,9 +54,9 @@ public AutomaticLogWriter( LogData.FieldGetterFactory fieldGetterFactory, byte[] jobId, ShuffleHelper helper, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, Duration writeLogstoreWatermarkGap) { - this.arcticLogWriter = + this.amoroLogWriter = new HiddenLogWriter( schema, producerConfig, topic, factory, fieldGetterFactory, jobId, helper); this.status = new AutomaticDoubleWriteStatus(tableLoader, writeLogstoreWatermarkGap); @@ -66,27 +66,27 @@ public AutomaticLogWriter( public void setup( StreamTask containingTask, StreamConfig config, Output> output) { super.setup(containingTask, config, output); - arcticLogWriter.setup(containingTask, config, output); + amoroLogWriter.setup(containingTask, config, output); status.setup(getRuntimeContext().getIndexOfThisSubtask()); } @Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); - arcticLogWriter.initializeState(context); + amoroLogWriter.initializeState(context); } @Override public void open() throws Exception { super.open(); - arcticLogWriter.open(); + amoroLogWriter.open(); status.open(); } @Override public void processElement(StreamRecord element) throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.processElement(element); + amoroLogWriter.processElement(element); } } @@ -99,7 +99,7 @@ public void processWatermark(Watermark mark) throws Exception { @Override public void prepareSnapshotPreBarrier(long checkpointId) throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.prepareSnapshotPreBarrier(checkpointId); + amoroLogWriter.prepareSnapshotPreBarrier(checkpointId); } else { status.sync(); } @@ -108,35 +108,35 @@ public void prepareSnapshotPreBarrier(long checkpointId) throws Exception { @Override public void snapshotState(StateSnapshotContext context) throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.snapshotState(context); + amoroLogWriter.snapshotState(context); } } @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.notifyCheckpointComplete(checkpointId); + amoroLogWriter.notifyCheckpointComplete(checkpointId); } } @Override public void notifyCheckpointAborted(long checkpointId) throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.notifyCheckpointAborted(checkpointId); + amoroLogWriter.notifyCheckpointAborted(checkpointId); } } @Override public void close() throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.close(); + amoroLogWriter.close(); } } @Override public void endInput() throws Exception { if (status.isDoubleWrite()) { - arcticLogWriter.endInput(); + amoroLogWriter.endInput(); } } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/FlinkSink.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/FlinkSink.java index edcabf9dd6..3eaabd0296 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/FlinkSink.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/FlinkSink.java @@ -38,9 +38,9 @@ import org.apache.amoro.flink.shuffle.ShuffleHelper; import org.apache.amoro.flink.shuffle.ShuffleKey; import org.apache.amoro.flink.shuffle.ShuffleRulePolicy; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.table.descriptors.ArcticValidator; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.util.CompatibleFlinkPropertyUtil; import org.apache.amoro.flink.util.IcebergClassUtil; import org.apache.amoro.flink.util.ProxyUtil; @@ -94,7 +94,7 @@ public static class Builder { private DataStream rowDataInput = null; private ProviderContext context; private MixedTable table; - private ArcticTableLoader tableLoader; + private AmoroTableLoader tableLoader; private TableSchema flinkSchema; private Properties producerConfig; private String topic; @@ -134,7 +134,7 @@ public Builder topic(String topic) { return this; } - public Builder tableLoader(ArcticTableLoader tableLoader) { + public Builder tableLoader(AmoroTableLoader tableLoader) { this.tableLoader = tableLoader; return this; } @@ -151,8 +151,8 @@ public Builder distribute(DistributionHashMode distributionMode) { DataStreamSink withEmit( DataStream input, - ArcticLogWriter logWriter, - ArcticFileWriter fileWriter, + AmoroLogWriter logWriter, + AmoroFileWriter fileWriter, OneInputStreamOperator committer, int writeOperatorParallelism, MetricsGenerator metricsGenerator, @@ -160,10 +160,10 @@ DataStreamSink withEmit( SingleOutputStreamOperator writerStream = input .transform( - ArcticWriter.class.getName(), + AmoroWriter.class.getName(), TypeExtractor.createTypeInfo(WriteResult.class), - new ArcticWriter<>(logWriter, fileWriter, metricsGenerator)) - .name(String.format("ArcticWriter %s(%s)", table.name(), arcticEmitMode)) + new AmoroWriter<>(logWriter, fileWriter, metricsGenerator)) + .name(String.format("AmoroWriter %s(%s)", table.name(), arcticEmitMode)) .setParallelism(writeOperatorParallelism); if (committer != null) { @@ -176,7 +176,7 @@ DataStreamSink withEmit( return writerStream .addSink(new DiscardingSink<>()) - .name(String.format("ArcticSink %s", table.name())) + .name(String.format("AmoroSink %s", table.name())) .setParallelism(1); } @@ -227,12 +227,12 @@ public DataStreamSink build() { final Duration watermarkWriteGap = config.get(AUTO_EMIT_LOGSTORE_WATERMARK_GAP); - ArcticFileWriter fileWriter = + AmoroFileWriter fileWriter = createFileWriter( table, shufflePolicy, overwrite, flinkSchemaRowType, arcticEmitMode, tableLoader); - ArcticLogWriter logWriter = - ArcticUtils.buildArcticLogWriter( + AmoroLogWriter logWriter = + AmoroUtils.buildArcticLogWriter( table.properties(), producerConfig, topic, @@ -243,7 +243,7 @@ public DataStreamSink build() { watermarkWriteGap); MetricsGenerator metricsGenerator = - ArcticUtils.getMetricsGenerator( + AmoroUtils.getMetricsGenerator( metricsEventLatency, metricsEnable, table, flinkSchemaRowType, writeSchema); if (shufflePolicy != null) { @@ -264,7 +264,7 @@ public DataStreamSink build() { private void initTableIfNeeded() { if (table == null) { - table = ArcticUtils.loadArcticTable(tableLoader); + table = AmoroUtils.loadArcticTable(tableLoader); } } @@ -331,7 +331,7 @@ public static ShuffleRulePolicy buildShuffleRulePolicy( + " for table without partition"); } int writeFileSplit; - if (ArcticUtils.isToBase(overwrite)) { + if (AmoroUtils.isToBase(overwrite)) { writeFileSplit = PropertyUtil.propertyAsInt( table.properties(), @@ -351,24 +351,24 @@ public static ShuffleRulePolicy buildShuffleRulePolicy( } } - public static ArcticFileWriter createFileWriter( + public static AmoroFileWriter createFileWriter( MixedTable mixedTable, ShuffleRulePolicy shufflePolicy, boolean overwrite, RowType flinkSchema, - ArcticTableLoader tableLoader) { + AmoroTableLoader tableLoader) { return createFileWriter( mixedTable, shufflePolicy, overwrite, flinkSchema, ARCTIC_EMIT_FILE, tableLoader); } - public static ArcticFileWriter createFileWriter( + public static AmoroFileWriter createFileWriter( MixedTable mixedTable, ShuffleRulePolicy shufflePolicy, boolean overwrite, RowType flinkSchema, String emitMode, - ArcticTableLoader tableLoader) { - if (!ArcticUtils.arcticFileWriterEnable(emitMode)) { + AmoroTableLoader tableLoader) { + if (!AmoroUtils.arcticFileWriterEnable(emitMode)) { return null; } long maxOpenFilesSizeBytes = @@ -398,7 +398,7 @@ public static ArcticFileWriter createFileWriter( SUBMIT_EMPTY_SNAPSHOTS.key(), SUBMIT_EMPTY_SNAPSHOTS.defaultValue()); - return new ArcticFileWriter( + return new AmoroFileWriter( shufflePolicy, createTaskWriterFactory(mixedTable, overwrite, flinkSchema), minFileSplitCount, @@ -409,12 +409,12 @@ public static ArcticFileWriter createFileWriter( private static TaskWriterFactory createTaskWriterFactory( MixedTable mixedTable, boolean overwrite, RowType flinkSchema) { - return new ArcticRowDataTaskWriterFactory(mixedTable, flinkSchema, overwrite); + return new AmoroRowDataTaskWriterFactory(mixedTable, flinkSchema, overwrite); } public static OneInputStreamOperator createFileCommitter( MixedTable mixedTable, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, boolean overwrite, String branch, PartitionSpec spec) { @@ -423,15 +423,15 @@ public static OneInputStreamOperator createFileCommitter( public static OneInputStreamOperator createFileCommitter( MixedTable mixedTable, - ArcticTableLoader tableLoader, + AmoroTableLoader tableLoader, boolean overwrite, String branch, PartitionSpec spec, String emitMode) { - if (!ArcticUtils.arcticFileWriterEnable(emitMode)) { + if (!AmoroUtils.arcticFileWriterEnable(emitMode)) { return null; } - tableLoader.switchLoadInternalTableForKeyedTable(ArcticUtils.isToBase(overwrite)); + tableLoader.switchLoadInternalTableForKeyedTable(AmoroUtils.isToBase(overwrite)); return (OneInputStreamOperator) ProxyUtil.getProxy( IcebergClassUtil.newIcebergFilesCommitter( diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AbstractHiddenLogWriter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AbstractHiddenLogWriter.java index cbefd45ce1..c9b96bc4e9 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AbstractHiddenLogWriter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AbstractHiddenLogWriter.java @@ -23,7 +23,7 @@ import org.apache.amoro.data.ChangeAction; import org.apache.amoro.flink.shuffle.LogRecordV1; import org.apache.amoro.flink.shuffle.ShuffleHelper; -import org.apache.amoro.flink.write.ArcticLogWriter; +import org.apache.amoro.flink.write.AmoroLogWriter; import org.apache.amoro.log.FormatVersion; import org.apache.amoro.log.LogData; import org.apache.amoro.log.LogDataJsonSerialization; @@ -52,7 +52,7 @@ * will process records after all operators has sent flip message to the jobMaster and the jobMaster * has finished handling these requests. */ -public abstract class AbstractHiddenLogWriter extends ArcticLogWriter { +public abstract class AbstractHiddenLogWriter extends AmoroLogWriter { public static final Logger LOG = LoggerFactory.getLogger(AbstractHiddenLogWriter.class); private static final long serialVersionUID = 1L; diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/ArcticLogPartitioner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AmoroLogPartitioner.java similarity index 94% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/ArcticLogPartitioner.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AmoroLogPartitioner.java index bfb3c209f7..e847d25cf7 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/ArcticLogPartitioner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/AmoroLogPartitioner.java @@ -31,12 +31,12 @@ import java.util.concurrent.atomic.AtomicInteger; /** This is a log message partitioner that makes sure the record is without out-of-order. */ -public class ArcticLogPartitioner implements Serializable { +public class AmoroLogPartitioner implements Serializable { private static final long serialVersionUID = 9184708069203854226L; private final AtomicInteger counter = new AtomicInteger(0); private final ShuffleHelper helper; - public ArcticLogPartitioner(ShuffleHelper shuffleHelper) { + public AmoroLogPartitioner(ShuffleHelper shuffleHelper) { this.helper = shuffleHelper; } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaFactory.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaFactory.java index ee927ee530..5eb1225c10 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaFactory.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaFactory.java @@ -21,7 +21,7 @@ import static org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull; import org.apache.amoro.flink.shuffle.ShuffleHelper; -import org.apache.amoro.flink.write.hidden.ArcticLogPartitioner; +import org.apache.amoro.flink.write.hidden.AmoroLogPartitioner; import org.apache.amoro.flink.write.hidden.LogMsgFactory; import org.apache.amoro.log.LogDataJsonSerialization; @@ -39,7 +39,7 @@ public Producer createProducer( ShuffleHelper helper) { checkNotNull(topic); return new HiddenKafkaProducer<>( - producerConfig, topic, logDataJsonSerialization, new ArcticLogPartitioner<>(helper)); + producerConfig, topic, logDataJsonSerialization, new AmoroLogPartitioner<>(helper)); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaProducer.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaProducer.java index ac937bc7f0..206953c912 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaProducer.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/main/java/org/apache/amoro/flink/write/hidden/kafka/HiddenKafkaProducer.java @@ -20,7 +20,7 @@ import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; -import org.apache.amoro.flink.write.hidden.ArcticLogPartitioner; +import org.apache.amoro.flink.write.hidden.AmoroLogPartitioner; import org.apache.amoro.flink.write.hidden.LogMsgFactory; import org.apache.amoro.log.LogData; import org.apache.amoro.log.LogDataJsonSerialization; @@ -67,18 +67,18 @@ public class HiddenKafkaProducer implements LogMsgFactory.Producer { private transient FlinkKafkaInternalProducer producer; private transient FlinkKafkaInternalProducer transactionalProducer; - private final ArcticLogPartitioner arcticLogPartitioner; + private final AmoroLogPartitioner amoroLogPartitioner; private int[] partitions; public HiddenKafkaProducer( Properties producerConfig, String topic, LogDataJsonSerialization logDataJsonSerialization, - ArcticLogPartitioner arcticLogPartitioner) { + AmoroLogPartitioner amoroLogPartitioner) { this.producerConfig = producerConfig; this.topic = topic; this.logDataJsonSerialization = logDataJsonSerialization; - this.arcticLogPartitioner = arcticLogPartitioner; + this.amoroLogPartitioner = amoroLogPartitioner; } @Override @@ -101,7 +101,7 @@ public void open() throws Exception { public void send(LogData logData) throws Exception { checkErroneous(); byte[] message = logDataJsonSerialization.serialize(logData); - int partition = arcticLogPartitioner.partition(logData, partitions); + int partition = amoroLogPartitioner.partition(logData, partitions); ProducerRecord producerRecord = new ProducerRecord<>(topic, partition, null, null, message); producer.send(producerRecord, callback); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTableTestBase.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTableTestBase.java index 1b348a9269..9766870c40 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTableTestBase.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTableTestBase.java @@ -18,8 +18,8 @@ package org.apache.amoro.flink; -import org.apache.amoro.flink.table.ArcticTableLoader; -import org.apache.amoro.flink.write.ArcticRowDataTaskWriterFactory; +import org.apache.amoro.flink.table.AmoroTableLoader; +import org.apache.amoro.flink.write.AmoroRowDataTaskWriterFactory; import org.apache.amoro.table.KeyedTable; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableIdentifier; @@ -64,8 +64,8 @@ default TaskWriter createUnkeyedTaskWriter(UnkeyedTable unkeyedTable, R default TaskWriter createTaskWriter( MixedTable mixedTable, RowType rowType, boolean overwrite, long mask) { - ArcticRowDataTaskWriterFactory taskWriterFactory = - new ArcticRowDataTaskWriterFactory(mixedTable, rowType, overwrite); + AmoroRowDataTaskWriterFactory taskWriterFactory = + new AmoroRowDataTaskWriterFactory(mixedTable, rowType, overwrite); taskWriterFactory.setMask(mask); taskWriterFactory.initialize(0, 0); return taskWriterFactory.create(); @@ -97,13 +97,13 @@ default void commit(MixedTable mixedTable, WriteResult result, boolean base) { } } - default ArcticTableLoader getTableLoader( + default AmoroTableLoader getTableLoader( String catalogName, String metastoreUrl, MixedTable mixedTable) { TableIdentifier identifier = TableIdentifier.of( catalogName, mixedTable.id().getDatabase(), mixedTable.id().getTableName()); InternalCatalogBuilder internalCatalogBuilder = InternalCatalogBuilder.builder().metastoreUrl(metastoreUrl); - return ArcticTableLoader.of(identifier, internalCatalogBuilder, mixedTable.properties()); + return AmoroTableLoader.of(identifier, internalCatalogBuilder, mixedTable.properties()); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java index c266b92897..c21a03b213 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/FlinkTestBase.java @@ -27,7 +27,7 @@ import org.apache.amoro.catalog.CatalogTestHelper; import org.apache.amoro.catalog.TableTestBase; import org.apache.amoro.flink.catalog.factories.CatalogFactoryOptions; -import org.apache.amoro.flink.write.ArcticRowDataTaskWriterFactory; +import org.apache.amoro.flink.write.AmoroRowDataTaskWriterFactory; import org.apache.amoro.io.reader.GenericKeyedDataReader; import org.apache.amoro.scan.CombinedScanTask; import org.apache.amoro.scan.KeyedTableScanTask; @@ -315,8 +315,8 @@ protected static TaskWriter createKeyedTaskWriter( protected static TaskWriter createKeyedTaskWriter( KeyedTable keyedTable, RowType rowType, boolean base, long mask) { - ArcticRowDataTaskWriterFactory taskWriterFactory = - new ArcticRowDataTaskWriterFactory(keyedTable, rowType, base); + AmoroRowDataTaskWriterFactory taskWriterFactory = + new AmoroRowDataTaskWriterFactory(keyedTable, rowType, base); taskWriterFactory.setMask(mask); taskWriterFactory.initialize(0, 0); return taskWriterFactory.create(); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestArcticSource.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestAmoroSource.java similarity index 92% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestArcticSource.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestAmoroSource.java index f3a4701a01..aab4eb0cfe 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestArcticSource.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestAmoroSource.java @@ -30,11 +30,11 @@ import org.apache.amoro.flink.read.hybrid.reader.ReaderFunction; import org.apache.amoro.flink.read.hybrid.reader.RowDataReaderFunction; import org.apache.amoro.flink.read.hybrid.reader.TestRowDataReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.source.ArcticScanContext; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.source.AmoroScanContext; import org.apache.amoro.flink.read.source.DataIterator; -import org.apache.amoro.flink.table.ArcticTableLoader; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.table.AmoroTableLoader; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.write.FlinkSink; import org.apache.amoro.mixed.MixedFormatCatalog; import org.apache.amoro.table.KeyedTable; @@ -104,8 +104,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; -public class TestArcticSource extends TestRowDataReaderFunction implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(TestArcticSource.class); +public class TestAmoroSource extends TestRowDataReaderFunction implements Serializable { + private static final Logger LOG = LoggerFactory.getLogger(TestAmoroSource.class); private static final long serialVersionUID = 7418812854449034756L; private static final int PARALLELISM = 1; @@ -155,14 +155,14 @@ public void dropTable() { @Test public void testArcticSourceStatic() throws Exception { - ArcticSource arcticSource = initArcticSource(false); + AmoroSource amoroSource = initArcticSource(false); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(3000); // set the source parallelism to 4 final CloseableIterator resultIterator = - env.fromSource(arcticSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") + env.fromSource(amoroSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") .setParallelism(PARALLELISM) .executeAndCollect(); @@ -196,14 +196,14 @@ public void testArcticSource(FailoverType failoverType) throws Exception { expected.addAll(updated); expected.addAll(records); - ArcticSource arcticSource = initArcticSource(false); + AmoroSource amoroSource = initArcticSource(false); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0)); DataStream input = - env.fromSource(arcticSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") + env.fromSource(amoroSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") .setParallelism(PARALLELISM); List expectedAfterMoR = new ArrayList<>(mor(expected)); @@ -213,7 +213,7 @@ public void testArcticSource(FailoverType failoverType) throws Exception { FlinkSink.forRowData(streamFailingInTheMiddleOfReading) .context(Optional::of) .table(testFailoverTable) - .tableLoader(ArcticTableLoader.of(FAIL_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(FAIL_TABLE_ID, catalogBuilder)) .flinkSchema(FLINK_SCHEMA) .build(); @@ -237,14 +237,14 @@ public void testDimTaskManagerFailover() throws Exception { List records = generateRecords(2, 1); writeUpdate(records); - ArcticSource arcticSource = initArcticDimSource(true); + AmoroSource amoroSource = initArcticDimSource(true); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, 0)); DataStream input = - env.fromSource(arcticSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") + env.fromSource(amoroSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") .setParallelism(PARALLELISM); WatermarkAwareFailWrapper.wrapWithFailureAfter(input); @@ -268,11 +268,11 @@ public void testDimTaskManagerFailover() throws Exception { @Test public void testArcticContinuousSource() throws Exception { - ArcticSource arcticSource = initArcticSource(true); + AmoroSource amoroSource = initArcticSource(true); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); - ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, arcticSource); + ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, amoroSource); JobClient jobClient = clientAndIterator.client; @@ -342,12 +342,11 @@ public void testArcticContinuousSourceWithEmptyChangeInInit() throws Exception { } commit(table, taskWriter.complete(), true); - ArcticSource arcticSource = - initArcticSource(true, SCAN_STARTUP_MODE_EARLIEST, tableId); + AmoroSource amoroSource = initArcticSource(true, SCAN_STARTUP_MODE_EARLIEST, tableId); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); - ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, arcticSource); + ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, amoroSource); JobClient jobClient = clientAndIterator.client; @@ -428,13 +427,13 @@ public void testArcticSourceEnumeratorWithChangeExpired() throws Exception { } final Duration monitorInterval = Duration.ofSeconds(1); - ArcticSource arcticSource = + AmoroSource amoroSource = initArcticSourceWithMonitorInterval( true, SCAN_STARTUP_MODE_EARLIEST, tableId, monitorInterval); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); - ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, arcticSource); + ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, amoroSource); JobClient jobClient = clientAndIterator.client; @@ -530,13 +529,13 @@ public void testArcticSourceEnumeratorWithBaseExpired() throws Exception { } final Duration monitorInterval = Duration.ofSeconds(1); - ArcticSource arcticSource = + AmoroSource amoroSource = initArcticSourceWithMonitorInterval( true, SCAN_STARTUP_MODE_EARLIEST, tableId, monitorInterval); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); - ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, arcticSource); + ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, amoroSource); JobClient jobClient = clientAndIterator.client; @@ -574,12 +573,12 @@ public void testArcticSourceEnumeratorWithBaseExpired() throws Exception { @Test public void testLatestStartupMode() throws Exception { - ArcticSource arcticSource = initArcticSourceWithLatest(); + AmoroSource amoroSource = initArcticSourceWithLatest(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); - ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, arcticSource); + ClientAndIterator clientAndIterator = executeAndCollectWithClient(env, amoroSource); JobClient jobClient = clientAndIterator.client; @@ -616,20 +615,20 @@ public void testArcticContinuousSource(final FailoverType failoverType) throws E writeUpdate(); expected.addAll(Arrays.asList(excepts2())); - ArcticSource arcticSource = initArcticSource(true); + AmoroSource amoroSource = initArcticSource(true); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // enable checkpoint env.enableCheckpointing(1000); // env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0)); DataStream input = - env.fromSource(arcticSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") + env.fromSource(amoroSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") .setParallelism(PARALLELISM); FlinkSink.forRowData(input) .context(Optional::of) .table(testFailoverTable) - .tableLoader(ArcticTableLoader.of(FAIL_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(FAIL_TABLE_ID, catalogBuilder)) .flinkSchema(FLINK_SCHEMA) .build(); @@ -682,7 +681,7 @@ private boolean equalsRecords(List expected, List tableRecords public static List tableRecords(final KeyedTable keyedTable) { keyedTable.refresh(); - List arcticSplits = + List arcticSplits = FlinkSplitPlanner.planFullTable(keyedTable, new AtomicInteger(0)); RowDataReaderFunction rowDataReaderFunction = @@ -840,9 +839,9 @@ public void run() { } private ClientAndIterator executeAndCollectWithClient( - StreamExecutionEnvironment env, ArcticSource arcticSource) throws Exception { + StreamExecutionEnvironment env, AmoroSource amoroSource) throws Exception { final DataStreamSource source = - env.fromSource(arcticSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") + env.fromSource(amoroSource, WatermarkStrategy.noWatermarks(), "ArcticParallelSource") .setParallelism(PARALLELISM); return DataStreamUtils.collectWithClient(source, "job_" + name.getMethodName()); } @@ -888,61 +887,56 @@ private static void expireSnapshots( LOG.info("to delete {} files, success delete {} files", toDeleteFiles.get(), deleteFiles.get()); } - private ArcticSource initArcticSource(boolean isStreaming) { + private AmoroSource initArcticSource(boolean isStreaming) { return initArcticSource(isStreaming, SCAN_STARTUP_MODE_EARLIEST); } - private ArcticSource initArcticSourceWithLatest() { + private AmoroSource initArcticSourceWithLatest() { return initArcticSource(true, SCAN_STARTUP_MODE_LATEST); } - private ArcticSource initArcticSource(boolean isStreaming, String scanStartupMode) { - ArcticTableLoader tableLoader = initLoader(); - ArcticScanContext arcticScanContext = initArcticScanContext(isStreaming, scanStartupMode); + private AmoroSource initArcticSource(boolean isStreaming, String scanStartupMode) { + AmoroTableLoader tableLoader = initLoader(); + AmoroScanContext amoroScanContext = initArcticScanContext(isStreaming, scanStartupMode); ReaderFunction rowDataReaderFunction = initRowDataReadFunction(); TypeInformation typeInformation = InternalTypeInfo.of(FlinkSchemaUtil.convert(testKeyedTable.schema())); - return new ArcticSource<>( + return new AmoroSource<>( tableLoader, - arcticScanContext, + amoroScanContext, rowDataReaderFunction, typeInformation, testKeyedTable.name(), false); } - private ArcticSource initArcticSourceWithMonitorInterval( + private AmoroSource initArcticSourceWithMonitorInterval( boolean isStreaming, String scanStartupMode, TableIdentifier tableIdentifier, Duration monitorInterval) { - ArcticTableLoader tableLoader = ArcticTableLoader.of(tableIdentifier, catalogBuilder); - ArcticScanContext arcticScanContext = + AmoroTableLoader tableLoader = AmoroTableLoader.of(tableIdentifier, catalogBuilder); + AmoroScanContext amoroScanContext = initArcticScanContext(isStreaming, scanStartupMode, monitorInterval); - MixedTable table = ArcticUtils.loadArcticTable(tableLoader); + MixedTable table = AmoroUtils.loadArcticTable(tableLoader); ReaderFunction rowDataReaderFunction = initRowDataReadFunction(table.asKeyedTable()); TypeInformation typeInformation = InternalTypeInfo.of(FlinkSchemaUtil.convert(table.schema())); - return new ArcticSource<>( - tableLoader, - arcticScanContext, - rowDataReaderFunction, - typeInformation, - table.name(), - false); + return new AmoroSource<>( + tableLoader, amoroScanContext, rowDataReaderFunction, typeInformation, table.name(), false); } - private ArcticSource initArcticSource( + private AmoroSource initArcticSource( boolean isStreaming, String scanStartupMode, TableIdentifier tableIdentifier) { return initArcticSourceWithMonitorInterval( isStreaming, scanStartupMode, tableIdentifier, Duration.ofMillis(500)); } - private ArcticSource initArcticDimSource(boolean isStreaming) { - ArcticTableLoader tableLoader = initLoader(); - ArcticScanContext arcticScanContext = + private AmoroSource initArcticDimSource(boolean isStreaming) { + AmoroTableLoader tableLoader = initLoader(); + AmoroScanContext amoroScanContext = initArcticScanContext(isStreaming, SCAN_STARTUP_MODE_EARLIEST); ReaderFunction rowDataReaderFunction = initRowDataReadFunction(); Schema schema = testKeyedTable.schema(); @@ -953,9 +947,9 @@ private ArcticSource initArcticDimSource(boolean isStreaming) { TypeInformation typeInformation = InternalTypeInfo.of(FlinkSchemaUtil.convert(schemaWithWm)); - return new ArcticSource<>( + return new AmoroSource<>( tableLoader, - arcticScanContext, + amoroScanContext, rowDataReaderFunction, typeInformation, testKeyedTable.name(), @@ -977,25 +971,25 @@ private RowDataReaderFunction initRowDataReadFunction(KeyedTable keyedTable) { keyedTable.io()); } - private ArcticScanContext initArcticScanContext( + private AmoroScanContext initArcticScanContext( boolean isStreaming, String scanStartupMode, Duration monitorInterval) { - return ArcticScanContext.arcticBuilder() + return AmoroScanContext.arcticBuilder() .streaming(isStreaming) .scanStartupMode(scanStartupMode) .monitorInterval(monitorInterval) .build(); } - private ArcticScanContext initArcticScanContext(boolean isStreaming, String scanStartupMode) { - return ArcticScanContext.arcticBuilder() + private AmoroScanContext initArcticScanContext(boolean isStreaming, String scanStartupMode) { + return AmoroScanContext.arcticBuilder() .streaming(isStreaming) .scanStartupMode(scanStartupMode) .monitorInterval(Duration.ofMillis(500)) .build(); } - private ArcticTableLoader initLoader() { - return ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + private AmoroTableLoader initLoader() { + return AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); } // ------------------------------------------------------------------------ diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java index 525bd304e5..2dd31cf273 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSource.java @@ -18,15 +18,15 @@ package org.apache.amoro.flink.read; -import static org.apache.amoro.flink.write.TestArcticFileWriter.TARGET_FILE_SIZE; -import static org.apache.amoro.flink.write.TestArcticFileWriter.createUnkeyedTaskWriter; +import static org.apache.amoro.flink.write.TestAmoroFileWriter.TARGET_FILE_SIZE; +import static org.apache.amoro.flink.write.TestAmoroFileWriter.createUnkeyedTaskWriter; import org.apache.amoro.BasicTableTestHelper; import org.apache.amoro.TableFormat; import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.table.FlinkSource; import org.apache.amoro.flink.util.DataUtil; import org.apache.flink.configuration.Configuration; @@ -127,7 +127,7 @@ public void testUnkeyedTableDataStream() throws Exception { .env(env) .context(Optional::of) .project(FLINK_SCHEMA) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkConf(conf) .properties( new HashMap() { @@ -177,7 +177,7 @@ public void testUnkeyedStreamingRead() throws Exception { .env(env) .context(Optional::of) .project(FLINK_SCHEMA) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkConf(conf) .build(); @@ -244,7 +244,7 @@ public void testUnkeyedSnapshotRead() throws Exception { .env(env) .context(Optional::of) .project(FLINK_SCHEMA) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkConf(conf) .properties( new HashMap() { @@ -279,7 +279,7 @@ public void testUnkeyedSnapshotRead() throws Exception { .env(env) .context(Optional::of) .project(FLINK_SCHEMA) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkConf(conf) .properties( new HashMap() { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java index 895961b4c0..3583d3789b 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/TestFlinkSplitPlanner.java @@ -19,7 +19,7 @@ package org.apache.amoro.flink.read; import org.apache.amoro.flink.read.hybrid.reader.TestRowDataReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.scan.ChangeTableIncrementalScan; import org.apache.iceberg.Snapshot; import org.junit.Assert; @@ -35,7 +35,7 @@ public class TestFlinkSplitPlanner extends TestRowDataReaderFunction { public void testPlanSplitFromKeyedTable() { testKeyedTable.baseTable().refresh(); testKeyedTable.changeTable().refresh(); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); Assert.assertEquals(7, splitList.size()); } @@ -44,7 +44,7 @@ public void testPlanSplitFromKeyedTable() { public void testIncrementalChangelog() throws IOException { testKeyedTable.baseTable().refresh(); testKeyedTable.changeTable().refresh(); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); Assert.assertEquals(7, splitList.size()); @@ -64,7 +64,7 @@ public void testIncrementalChangelog() throws IOException { .useSnapshot(nowSnapshotId) .fromSequence(fromSequence); - List changeSplits = + List changeSplits = FlinkSplitPlanner.planChangeTable(changeTableScan, new AtomicInteger()); Assert.assertEquals(1, changeSplits.size()); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java index 47cbade07f..1a81c4a1f5 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestShuffleSplitAssigner.java @@ -22,7 +22,7 @@ import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.reader.RowDataReaderFunction; import org.apache.amoro.flink.read.hybrid.reader.TestRowDataReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.source.DataIterator; import org.apache.flink.api.connector.source.ReaderInfo; import org.apache.flink.api.connector.source.SourceEvent; @@ -53,10 +53,10 @@ public class TestShuffleSplitAssigner extends TestRowDataReaderFunction { public void testSingleParallelism() { ShuffleSplitAssigner shuffleSplitAssigner = instanceSplitAssigner(1); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); shuffleSplitAssigner.onDiscoveredSplits(splitList); - List actual = new ArrayList<>(); + List actual = new ArrayList<>(); while (true) { Split splitOpt = shuffleSplitAssigner.getNext(0); @@ -74,10 +74,10 @@ public void testSingleParallelism() { public void testMultiParallelism() { ShuffleSplitAssigner shuffleSplitAssigner = instanceSplitAssigner(3); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); shuffleSplitAssigner.onDiscoveredSplits(splitList); - List actual = new ArrayList<>(); + List actual = new ArrayList<>(); int subtaskId = 2; while (subtaskId >= 0) { @@ -110,8 +110,8 @@ public void testTreeNodeMaskUpdate() { List actualNodes = new ArrayList<>(); for (long[] node : treeNodes) { - ArcticSplit arcticSplit = - new ArcticSplit() { + AmoroSplit arcticSplit = + new AmoroSplit() { DataTreeNode dataTreeNode = DataTreeNode.of(node[0], node[1]); @Override @@ -123,7 +123,7 @@ public Integer taskIndex() { public void updateOffset(Object[] recordOffsets) {} @Override - public ArcticSplit copy() { + public AmoroSplit copy() { return null; } @@ -161,7 +161,7 @@ public String toString() { @Test public void testNodeUpMoved() throws IOException { writeUpdateWithSpecifiedMaskOne(); - List arcticSplits = + List arcticSplits = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger(0)); int totalParallelism = 3; ShuffleSplitAssigner assigner = instanceSplitAssigner(totalParallelism); @@ -206,13 +206,13 @@ public void testNodeUpMoved() throws IOException { } protected ShuffleSplitAssigner instanceSplitAssigner(int parallelism) { - SplitEnumeratorContext splitEnumeratorContext = + SplitEnumeratorContext splitEnumeratorContext = new InternalSplitEnumeratorContext(parallelism); return new ShuffleSplitAssigner(splitEnumeratorContext); } protected static class InternalSplitEnumeratorContext - implements SplitEnumeratorContext { + implements SplitEnumeratorContext { private final int parallelism; public InternalSplitEnumeratorContext(int parallelism) { @@ -238,7 +238,7 @@ public Map registeredReaders() { } @Override - public void assignSplits(SplitsAssignment newSplitAssignments) {} + public void assignSplits(SplitsAssignment newSplitAssignments) {} @Override public void signalNoMoreSplits(int subtask) {} diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java index 7b84ec5cc1..77c9a89c9f 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestSplitAssignerAwaiting.java @@ -19,8 +19,8 @@ package org.apache.amoro.flink.read.hybrid.assigner; import org.apache.amoro.flink.read.FlinkSplitPlanner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.junit.Assert; import org.junit.Test; @@ -43,7 +43,7 @@ public void testEmpty() { @Test public void testStaticAssign() { ShuffleSplitAssigner splitAssigner = instanceSplitAssigner(1); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); splitAssigner.onDiscoveredSplits(splitList); @@ -60,11 +60,11 @@ public void testContinueAssign() { ShuffleSplitAssigner assigner = instanceSplitAssigner(1); assertGetNext(assigner, Split.Status.UNAVAILABLE); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); - List splits1 = splitList.subList(0, 1); + List splits1 = splitList.subList(0, 1); assertAvailableFuture(assigner, () -> assigner.onDiscoveredSplits(splits1)); - List splits2 = splitList.subList(1, 2); + List splits2 = splitList.subList(1, 2); assertAvailableFuture(assigner, () -> assigner.onUnassignedSplits(splits2)); assigner.onDiscoveredSplits(splitList.subList(2, 4)); @@ -120,7 +120,7 @@ private void assertGetNext(ShuffleSplitAssigner assigner, Split.Status expectedS } private void assertSnapshot(ShuffleSplitAssigner assigner, int splitCount) { - Collection stateBeforeGet = assigner.state(); + Collection stateBeforeGet = assigner.state(); Assert.assertEquals(splitCount, stateBeforeGet.size()); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java index 6756bca00c..4cd54e0926 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/assigner/TestStaticSplitAssigner.java @@ -20,7 +20,7 @@ import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.reader.TestRowDataReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -38,11 +38,11 @@ public class TestStaticSplitAssigner extends TestRowDataReaderFunction { @Test public void testSingleParallelism() throws IOException { try (StaticSplitAssigner staticSplitAssigner = instanceStaticSplitAssigner()) { - List splitList = + List splitList = FlinkSplitPlanner.mergeOnReadPlan( testKeyedTable, Collections.emptyList(), new AtomicInteger()); staticSplitAssigner.onDiscoveredSplits(splitList); - List actual = new ArrayList<>(); + List actual = new ArrayList<>(); while (true) { Split splitOpt = staticSplitAssigner.getNext(0); @@ -60,11 +60,11 @@ public void testSingleParallelism() throws IOException { @Test public void testMultiParallelism() throws IOException { try (StaticSplitAssigner staticSplitAssigner = instanceStaticSplitAssigner()) { - List splitList = + List splitList = FlinkSplitPlanner.mergeOnReadPlan( testKeyedTable, Collections.emptyList(), new AtomicInteger()); staticSplitAssigner.onDiscoveredSplits(splitList); - List actual = new ArrayList<>(); + List actual = new ArrayList<>(); int subtaskId = 2; while (subtaskId >= 0) { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestArcticSourceEnumStateSerializer.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestAmoroSourceEnumStateSerializer.java similarity index 83% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestArcticSourceEnumStateSerializer.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestAmoroSourceEnumStateSerializer.java index bdf8635bc4..a3b947aec8 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestArcticSourceEnumStateSerializer.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestAmoroSourceEnumStateSerializer.java @@ -22,7 +22,7 @@ import org.apache.amoro.flink.read.hybrid.assigner.ShuffleSplitAssigner; import org.apache.amoro.flink.read.hybrid.assigner.Split; import org.apache.amoro.flink.read.hybrid.assigner.TestShuffleSplitAssigner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.junit.Assert; @@ -36,44 +36,44 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -public class TestArcticSourceEnumStateSerializer extends TestShuffleSplitAssigner { +public class TestAmoroSourceEnumStateSerializer extends TestShuffleSplitAssigner { private static final Logger LOG = - LoggerFactory.getLogger(TestArcticSourceEnumStateSerializer.class); + LoggerFactory.getLogger(TestAmoroSourceEnumStateSerializer.class); @Test public void testArcticEnumState() throws IOException { ShuffleSplitAssigner shuffleSplitAssigner = instanceSplitAssigner(3); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); shuffleSplitAssigner.onDiscoveredSplits(splitList); TemporalJoinSplits splits = new TemporalJoinSplits(splitList, null); - ArcticSourceEnumState expect = - new ArcticSourceEnumState( + AmoroSourceEnumState expect = + new AmoroSourceEnumState( shuffleSplitAssigner.state(), null, shuffleSplitAssigner.serializePartitionIndex(), splits); - ArcticSourceEnumStateSerializer arcticSourceEnumStateSerializer = - new ArcticSourceEnumStateSerializer(); + AmoroSourceEnumStateSerializer arcticSourceEnumStateSerializer = + new AmoroSourceEnumStateSerializer(); byte[] ser = arcticSourceEnumStateSerializer.serialize(expect); Assert.assertNotNull(ser); - ArcticSourceEnumState actual = arcticSourceEnumStateSerializer.deserialize(1, ser); + AmoroSourceEnumState actual = arcticSourceEnumStateSerializer.deserialize(1, ser); Assert.assertEquals(expect.pendingSplits().size(), actual.pendingSplits().size()); Assert.assertEquals( Objects.requireNonNull(expect.shuffleSplitRelation()).length, Objects.requireNonNull(actual.shuffleSplitRelation()).length); - SplitEnumeratorContext splitEnumeratorContext = + SplitEnumeratorContext splitEnumeratorContext = new InternalSplitEnumeratorContext(3); try (ShuffleSplitAssigner actualAssigner = new ShuffleSplitAssigner(splitEnumeratorContext, getMixedTable().name(), actual)) { - List actualSplits = new ArrayList<>(); + List actualSplits = new ArrayList<>(); int subtaskId = 2; while (subtaskId >= 0) { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestArcticSourceEnumerator.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestAmoroSourceEnumerator.java similarity index 88% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestArcticSourceEnumerator.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestAmoroSourceEnumerator.java index 2c0c12e680..2d6baf12ff 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestArcticSourceEnumerator.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestAmoroSourceEnumerator.java @@ -27,11 +27,11 @@ import org.apache.amoro.flink.FlinkTestBase; import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.assigner.ShuffleSplitAssigner; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplitState; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplitState; import org.apache.amoro.flink.read.hybrid.split.SplitRequestEvent; -import org.apache.amoro.flink.read.source.ArcticScanContext; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.read.source.AmoroScanContext; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.table.KeyedTable; import org.apache.flink.api.connector.source.ReaderInfo; import org.apache.flink.api.connector.source.SourceEvent; @@ -62,9 +62,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; -public class TestArcticSourceEnumerator extends FlinkTestBase { +public class TestAmoroSourceEnumerator extends FlinkTestBase { - public TestArcticSourceEnumerator() { + public TestAmoroSourceEnumerator() { super( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); @@ -130,27 +130,27 @@ public void testReadersNumGreaterThanSplits() throws Exception { TestingSplitEnumeratorContext splitEnumeratorContext = instanceSplitEnumeratorContext(parallelism); ShuffleSplitAssigner shuffleSplitAssigner = instanceSplitAssigner(splitEnumeratorContext); - ArcticScanContext scanContext = - ArcticScanContext.arcticBuilder() + AmoroScanContext scanContext = + AmoroScanContext.arcticBuilder() .streaming(true) .scanStartupMode(SCAN_STARTUP_MODE_EARLIEST) .build(); - List splitList = + List splitList = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger()); shuffleSplitAssigner.onDiscoveredSplits(splitList); assertSnapshot(shuffleSplitAssigner, splitCount); - ArcticSourceEnumerator enumerator = - new ArcticSourceEnumerator( + AmoroSourceEnumerator enumerator = + new AmoroSourceEnumerator( splitEnumeratorContext, shuffleSplitAssigner, - ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder), + AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder), scanContext, null, false); - Collection pendingSplitsEmpty = enumerator.snapshotState(1).pendingSplits(); + Collection pendingSplitsEmpty = enumerator.snapshotState(1).pendingSplits(); Assert.assertEquals(splitCount, pendingSplitsEmpty.size()); // register readers, and let them request a split @@ -181,7 +181,7 @@ public void testReadersNumGreaterThanSplits() throws Exception { } private void assertSnapshot(ShuffleSplitAssigner assigner, int splitCount) { - Collection stateBeforeGet = assigner.state(); + Collection stateBeforeGet = assigner.state(); Assert.assertEquals(splitCount, stateBeforeGet.size()); } @@ -195,17 +195,17 @@ private TestingSplitEnumeratorContext instanceSplitEnumeratorContext(int paralle } protected static class TestingSplitEnumeratorContext - implements SplitEnumeratorContext { + implements SplitEnumeratorContext { private final int parallelism; - private final HashMap> splitAssignments = + private final HashMap> splitAssignments = new HashMap<>(); private final HashMap> events = new HashMap<>(); private final HashMap registeredReaders = new HashMap<>(); - public Map> getSplitAssignments() { + public Map> getSplitAssignments() { return splitAssignments; } @@ -245,10 +245,10 @@ public Map registeredReaders() { } @Override - public void assignSplits(SplitsAssignment newSplitAssignments) { - for (final Map.Entry> entry : + public void assignSplits(SplitsAssignment newSplitAssignments) { + for (final Map.Entry> entry : newSplitAssignments.assignment().entrySet()) { - final SplitAssignmentState assignment = + final SplitAssignmentState assignment = splitAssignments.computeIfAbsent(entry.getKey(), (key) -> new SplitAssignmentState<>()); assignment.getAssignedSplits().addAll(entry.getValue()); @@ -256,7 +256,7 @@ public void assignSplits(SplitsAssignment newSplitAssignments) { } @Override - public void assignSplit(ArcticSplit split, int subtask) { + public void assignSplit(AmoroSplit split, int subtask) { SplitEnumeratorContext.super.assignSplit(split, subtask); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java index 2e0d3c753a..5f88fc2b35 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/enumerator/TestTemporalJoinSplitsThreadSafe.java @@ -18,7 +18,7 @@ package org.apache.amoro.flink.read.hybrid.enumerator; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.TemporalJoinSplits; import org.junit.Assert; import org.junit.Test; @@ -41,15 +41,15 @@ public void testTemporalJoinSplits() { allSplit.add(UUID.randomUUID().toString()); } - Collection arcticSplits = - allSplit.stream().map(TestArcticSplit::of).collect(Collectors.toList()); + Collection arcticSplits = + allSplit.stream().map(TestAmoroSplit::of).collect(Collectors.toList()); for (int i = 0; i < 2; i++) { round(allSplit, arcticSplits); } } - public void round(List allSplit, Collection arcticSplits) { + public void round(List allSplit, Collection arcticSplits) { TemporalJoinSplits temporalJoinSplits = new TemporalJoinSplits(arcticSplits, null); int n = allSplit.size(); @@ -58,11 +58,11 @@ public void round(List allSplit, Collection arcticSplits) { Collections.shuffle(s1); Collections.shuffle(s2); - List as = new ArrayList<>(arcticSplits); + List as = new ArrayList<>(arcticSplits); Collections.shuffle(as); int an = as.size(); - List as1 = new ArrayList<>(as.subList(0, (int) (2.0 / 3 * an))); - List as2 = new ArrayList<>(as.subList((int) (1.0 / 3 * an), an)); + List as1 = new ArrayList<>(as.subList(0, (int) (2.0 / 3 * an))); + List as2 = new ArrayList<>(as.subList((int) (1.0 / 3 * an), an)); CompletableFuture f1 = CompletableFuture.runAsync(() -> temporalJoinSplits.removeAndReturnIfAllFinished(s1)); CompletableFuture f2 = @@ -75,15 +75,15 @@ public void round(List allSplit, Collection arcticSplits) { Assert.assertTrue(temporalJoinSplits.removeAndReturnIfAllFinished(allSplit)); } - static class TestArcticSplit extends ArcticSplit { + static class TestAmoroSplit extends AmoroSplit { private final String splitId; - public TestArcticSplit(String splitId) { + public TestAmoroSplit(String splitId) { this.splitId = splitId; } - public static TestArcticSplit of(String splitId) { - return new TestArcticSplit(splitId); + public static TestAmoroSplit of(String splitId) { + return new TestAmoroSplit(splitId); } @Override @@ -95,8 +95,8 @@ public Integer taskIndex() { public void updateOffset(Object[] recordOffsets) {} @Override - public ArcticSplit copy() { - return new TestArcticSplit(splitId); + public AmoroSplit copy() { + return new TestAmoroSplit(splitId); } @Override diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java index d0fa906c19..f8beaa8df7 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/MixedIncrementalLoaderTest.java @@ -25,7 +25,7 @@ import org.apache.amoro.flink.read.MixedIncrementalLoader; import org.apache.amoro.flink.read.hybrid.enumerator.ContinuousSplitPlanner; import org.apache.amoro.flink.read.hybrid.enumerator.MergeOnReadIncrementalPlanner; -import org.apache.amoro.flink.read.source.FlinkArcticMORDataReader; +import org.apache.amoro.flink.read.source.FlinkAmoroMORDataReader; import org.apache.amoro.flink.util.DataUtil; import org.apache.amoro.flink.write.FlinkTaskWriterBaseTest; import org.apache.amoro.table.KeyedTable; @@ -118,8 +118,8 @@ public void testMOR() { new MergeOnReadIncrementalPlanner( getTableLoader(getCatalogName(), getMetastoreUrl(), keyedTable)); - FlinkArcticMORDataReader flinkArcticMORDataReader = - new FlinkArcticMORDataReader( + FlinkAmoroMORDataReader flinkAmoroMORDataReader = + new FlinkAmoroMORDataReader( keyedTable.io(), keyedTable.schema(), keyedTable.schema(), @@ -132,7 +132,7 @@ public void testMOR() { MixedIncrementalLoader incrementalLoader = new MixedIncrementalLoader<>( morPlanner, - flinkArcticMORDataReader, + flinkAmoroMORDataReader, new RowDataReaderFunction( new Configuration(), keyedTable.schema(), diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java index 8d07206be5..72ace97733 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java @@ -24,7 +24,7 @@ import org.apache.amoro.data.DataFileType; import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.enumerator.TestContinuousSplitPlannerImpl; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.hybrid.split.ChangelogSplit; import org.apache.amoro.flink.read.source.DataIterator; import org.apache.amoro.scan.ChangeTableIncrementalScan; @@ -71,7 +71,7 @@ public TestRowDataReaderFunction() { @Test public void testReadChangelog() throws IOException { - List arcticSplits = + List arcticSplits = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger(0)); RowDataReaderFunction rowDataReaderFunction = @@ -143,7 +143,7 @@ public void testReadChangelog() throws IOException { @Test public void testReadNodesUpMoved() throws IOException { writeUpdateWithSpecifiedMaskOne(); - List arcticSplits = + List arcticSplits = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger(0)); RowDataReaderFunction rowDataReaderFunction = diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestArcticSplitSerializer.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestAmoroSplitSerializer.java similarity index 83% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestArcticSplitSerializer.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestAmoroSplitSerializer.java index 96afec6799..5b7b5b460d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestArcticSplitSerializer.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/read/hybrid/split/TestAmoroSplitSerializer.java @@ -29,24 +29,24 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -public class TestArcticSplitSerializer extends TestRowDataReaderFunction { +public class TestAmoroSplitSerializer extends TestRowDataReaderFunction { @Test public void testSerAndDes() { - List arcticSplits = + List arcticSplits = FlinkSplitPlanner.planFullTable(testKeyedTable, new AtomicInteger(0)); assertSerializedSplitEquals(arcticSplits); } @Test public void testSerAndDesMoRSplit() { - List arcticSplits = + List arcticSplits = FlinkSplitPlanner.mergeOnReadPlan(testKeyedTable, null, new AtomicInteger(0)); assertSerializedSplitEquals(arcticSplits); } - private void assertSerializedSplitEquals(List expected) { - ArcticSplitSerializer serializer = new ArcticSplitSerializer(); + private void assertSerializedSplitEquals(List expected) { + AmoroSplitSerializer serializer = new AmoroSplitSerializer(); List contents = expected.stream() .map( @@ -61,7 +61,7 @@ private void assertSerializedSplitEquals(List expected) { .collect(Collectors.toList()); Assert.assertArrayEquals( - expected.toArray(new ArcticSplit[0]), + expected.toArray(new AmoroSplit[0]), contents.stream() .map( data -> { @@ -74,15 +74,15 @@ private void assertSerializedSplitEquals(List expected) { throw new FlinkRuntimeException(e); } }) - .toArray(ArcticSplit[]::new)); + .toArray(AmoroSplit[]::new)); } @Test public void testNullableSplit() throws IOException { - ArcticSplitSerializer serializer = new ArcticSplitSerializer(); + AmoroSplitSerializer serializer = new AmoroSplitSerializer(); byte[] ser = serializer.serialize(null); - ArcticSplit actual = serializer.deserialize(1, ser); + AmoroSplit actual = serializer.deserialize(1, ser); Assert.assertNull(actual); } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java index 93fd6a3412..acd9f2d626 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java @@ -25,7 +25,7 @@ import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.util.DataUtil; import org.apache.amoro.flink.util.TestUtil; import org.apache.amoro.table.KeyedTable; @@ -198,7 +198,7 @@ public void testLookupJoin() throws Exception { .build(); RowType rowType = (RowType) flinkSchema.toRowDataType().getLogicalType(); KeyedTable keyedTable = - (KeyedTable) ArcticUtils.loadArcticTable(ArcticTableLoader.of(TABLE_ID, catalogBuilder)); + (KeyedTable) AmoroUtils.loadArcticTable(AmoroTableLoader.of(TABLE_ID, catalogBuilder)); TaskWriter taskWriter = createKeyedTaskWriter(keyedTable, rowType, true); List baseData = new ArrayList() { @@ -292,7 +292,7 @@ public void testLookupJoinWithPartialFields() throws Exception { .build(); RowType rowType = (RowType) flinkSchema.toRowDataType().getLogicalType(); KeyedTable keyedTable = - (KeyedTable) ArcticUtils.loadArcticTable(ArcticTableLoader.of(TABLE_ID, catalogBuilder)); + (KeyedTable) AmoroUtils.loadArcticTable(AmoroTableLoader.of(TABLE_ID, catalogBuilder)); TaskWriter taskWriter = createKeyedTaskWriter(keyedTable, rowType, true); List baseData = new ArrayList() { diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java index c51f7baf6e..0e985cbb60 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestTableRefresh.java @@ -67,8 +67,8 @@ public static Collection parameters() { @Test public void testRefresh() { - ArcticTableLoader tableLoader = - ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + AmoroTableLoader tableLoader = + AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); tableLoader.open(); MixedTable mixedTable = tableLoader.loadArcticTable(); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java index 91abf6aa8f..14d1023c42 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestWatermark.java @@ -25,7 +25,7 @@ import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.util.DataUtil; import org.apache.amoro.flink.util.TestUtil; import org.apache.amoro.table.KeyedTable; @@ -120,8 +120,8 @@ public void testWatermark() throws Exception { RowType rowType = (RowType) flinkSchema.toRowDataType().getLogicalType(); KeyedTable keyedTable = (KeyedTable) - ArcticUtils.loadArcticTable( - ArcticTableLoader.of( + AmoroUtils.loadArcticTable( + AmoroTableLoader.of( TableIdentifier.of(TEST_CATALOG_NAME, DB, TABLE), catalogBuilder)); TaskWriter taskWriter = createKeyedTaskWriter(keyedTable, rowType, true); List baseData = @@ -184,8 +184,8 @@ public void testSelectWatermarkField() throws Exception { RowType rowType = (RowType) flinkSchema.toRowDataType().getLogicalType(); KeyedTable keyedTable = (KeyedTable) - ArcticUtils.loadArcticTable( - ArcticTableLoader.of( + AmoroUtils.loadArcticTable( + AmoroTableLoader.of( TableIdentifier.of(TEST_CATALOG_NAME, DB, TABLE), catalogBuilder)); TaskWriter taskWriter = createKeyedTaskWriter(keyedTable, rowType, true); List baseData = diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentArctic.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentAmoro.java similarity index 96% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentArctic.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentAmoro.java index 98f216a656..f9e76f4689 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentArctic.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentAmoro.java @@ -34,9 +34,9 @@ import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo; import org.apache.flink.util.UserCodeClassLoader; -public class MockEnvironmentArctic extends MockEnvironment { +public class MockEnvironmentAmoro extends MockEnvironment { - protected MockEnvironmentArctic( + protected MockEnvironmentAmoro( JobID jobID, JobVertexID jobVertexID, String taskName, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentBuilder.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentBuilder.java index bf45dc34d4..9b48375bd3 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentBuilder.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/util/MockEnvironmentBuilder.java @@ -185,7 +185,7 @@ public MockEnvironment build() { this.ioManager = new IOManagerAsync(); } - return new MockEnvironmentArctic( + return new MockEnvironmentAmoro( this.jobID, this.jobVertexID, this.taskName, diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/ArcticFileWriterITCase.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/AmoroFileWriterITCase.java similarity index 93% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/ArcticFileWriterITCase.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/AmoroFileWriterITCase.java index 63d8055ae8..2bb5ea2876 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/ArcticFileWriterITCase.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/AmoroFileWriterITCase.java @@ -24,9 +24,9 @@ import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.data.FileNameRules; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.read.TestArcticSource; -import org.apache.amoro.flink.table.ArcticTableLoader; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.read.TestAmoroSource; +import org.apache.amoro.flink.table.AmoroTableLoader; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.table.KeyedTable; import org.apache.amoro.table.MixedTable; import org.apache.flink.api.common.RuntimeExecutionMode; @@ -74,17 +74,17 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; -public class ArcticFileWriterITCase extends FlinkTestBase { +public class AmoroFileWriterITCase extends FlinkTestBase { - public static final Logger LOG = LoggerFactory.getLogger(ArcticFileWriterITCase.class); + public static final Logger LOG = LoggerFactory.getLogger(AmoroFileWriterITCase.class); private static final Map LATCH_MAP = new ConcurrentHashMap<>(); - public ArcticTableLoader tableLoader; + public AmoroTableLoader tableLoader; private String latchId; private final int NUM_SOURCES = 4; private final int NUM_RECORDS = 10000; - public ArcticFileWriterITCase() { + public AmoroFileWriterITCase() { super( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); @@ -220,7 +220,7 @@ public void cancel() { } protected JobGraph createJobGraph( - ArcticTableLoader tableLoader, TableSchema tableSchema, boolean triggerFailover) { + AmoroTableLoader tableLoader, TableSchema tableSchema, boolean triggerFailover) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); Configuration config = new Configuration(); config.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.STREAMING); @@ -237,7 +237,7 @@ protected JobGraph createJobGraph( DataStreamSource source = env.addSource(new StreamingExecutionTestSource(latchId, NUM_RECORDS, triggerFailover)) .setParallelism(4); - MixedTable table = ArcticUtils.loadArcticTable(tableLoader); + MixedTable table = AmoroUtils.loadArcticTable(tableLoader); FlinkSink.forRowData(source) .context(Optional::of) .table(table) @@ -251,7 +251,7 @@ protected JobGraph createJobGraph( @Test public void testWrite() throws Exception { - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); JobGraph jobGraph = createJobGraph(tableLoader, FLINK_SCHEMA, true); final Configuration config = new Configuration(); @@ -306,6 +306,6 @@ public static void checkResult(KeyedTable keyedTable, int exceptedSize) { maxTxId = maxTxIdInSnapshot; } - Assert.assertEquals(exceptedSize, TestArcticSource.tableRecords(keyedTable).size()); + Assert.assertEquals(exceptedSize, TestAmoroSource.tableRecords(keyedTable).size()); } } diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java index a9f73fd043..711ae1dbc6 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/FlinkTaskWriterBaseTest.java @@ -23,7 +23,7 @@ import org.apache.amoro.flink.FlinkTableTestBase; import org.apache.amoro.flink.read.FlinkSplitPlanner; import org.apache.amoro.flink.read.hybrid.reader.RowDataReaderFunction; -import org.apache.amoro.flink.read.hybrid.split.ArcticSplit; +import org.apache.amoro.flink.read.hybrid.split.AmoroSplit; import org.apache.amoro.flink.read.source.DataIterator; import org.apache.amoro.io.AuthenticatedFileIO; import org.apache.amoro.table.KeyedTable; @@ -138,7 +138,7 @@ default List recordsOfUnkeyedTable( default List recordsOfKeyedTable( KeyedTable table, Schema tableSchema, Schema projectedSchema, AuthenticatedFileIO io) { - List arcticSplits = FlinkSplitPlanner.planFullTable(table, new AtomicInteger(0)); + List arcticSplits = FlinkSplitPlanner.planFullTable(table, new AtomicInteger(0)); RowDataReaderFunction rowDataReaderFunction = new RowDataReaderFunction( diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestArcticFileCommitter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAmoroFileCommitter.java similarity index 90% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestArcticFileCommitter.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAmoroFileCommitter.java index 3474b756be..1640595831 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestArcticFileCommitter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAmoroFileCommitter.java @@ -23,8 +23,8 @@ import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.table.ArcticTableLoader; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.table.AmoroTableLoader; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.table.KeyedTable; import org.apache.amoro.table.MixedTable; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; @@ -43,17 +43,17 @@ import java.util.List; -public class TestArcticFileCommitter extends FlinkTestBase { - public ArcticTableLoader tableLoader; +public class TestAmoroFileCommitter extends FlinkTestBase { + public AmoroTableLoader tableLoader; - public TestArcticFileCommitter() { + public TestAmoroFileCommitter() { super( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(true, true)); } public OneInputStreamOperatorTestHarness createArcticFileCommitter( - ArcticTableLoader tableLoader, MixedTable table, OperatorSubtaskState operatorSubtaskState) + AmoroTableLoader tableLoader, MixedTable table, OperatorSubtaskState operatorSubtaskState) throws Exception { OneInputStreamOperator committer = FlinkSink.createFileCommitter( @@ -88,8 +88,8 @@ public void checkChangeFiles(int fileCnt, int recordCnt, KeyedTable table) { @Test public void testCommit() throws Exception { - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); - KeyedTable table = ArcticUtils.loadArcticTable(tableLoader).asKeyedTable(); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + KeyedTable table = AmoroUtils.loadArcticTable(tableLoader).asKeyedTable(); List completedFiles = prepareChangeFiles(); OperatorSubtaskState snapshot; @@ -115,7 +115,7 @@ private List prepareChangeFiles() throws Exception { List changeFiles; long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = - TestArcticFileWriter.createArcticStreamWriter(tableLoader)) { + TestAmoroFileWriter.createArcticStreamWriter(tableLoader)) { // The first checkpoint testHarness.processElement(createRowData(1, "hello", "2020-10-11T10:10:11.0"), 1); testHarness.processElement(createRowData(2, "hello", "2020-10-12T10:10:11.0"), 1); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestArcticFileWriter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAmoroFileWriter.java similarity index 92% rename from amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestArcticFileWriter.java rename to amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAmoroFileWriter.java index 9b1a0a6524..49c10197ed 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestArcticFileWriter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAmoroFileWriter.java @@ -25,7 +25,7 @@ import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.util.TestGlobalAggregateManager; import org.apache.amoro.flink.util.TestOneInputStreamOperatorIntern; import org.apache.amoro.table.MixedTable; @@ -50,10 +50,10 @@ import java.util.List; @RunWith(Parameterized.class) -public class TestArcticFileWriter extends FlinkTestBase { +public class TestAmoroFileWriter extends FlinkTestBase { public static final long TARGET_FILE_SIZE = 128 * 1024 * 1024; - public ArcticTableLoader tableLoader; + public AmoroTableLoader tableLoader; private final boolean submitEmptySnapshots; @Parameterized.Parameters(name = "{0}, {1}") @@ -66,7 +66,7 @@ public static Object[][] parameters() { }; } - public TestArcticFileWriter(boolean isKeyed, boolean submitEmptySnapshots) { + public TestAmoroFileWriter(boolean isKeyed, boolean submitEmptySnapshots) { super( new BasicCatalogTestHelper(TableFormat.MIXED_ICEBERG), new BasicTableTestHelper(isKeyed, true)); @@ -74,12 +74,12 @@ public TestArcticFileWriter(boolean isKeyed, boolean submitEmptySnapshots) { } public static OneInputStreamOperatorTestHarness createArcticStreamWriter( - ArcticTableLoader tableLoader) throws Exception { + AmoroTableLoader tableLoader) throws Exception { return createArcticStreamWriter(tableLoader, true, null); } public static OneInputStreamOperatorTestHarness createArcticStreamWriter( - ArcticTableLoader tableLoader, boolean submitEmptySnapshots, Long restoredCheckpointId) + AmoroTableLoader tableLoader, boolean submitEmptySnapshots, Long restoredCheckpointId) throws Exception { OneInputStreamOperatorTestHarness harness = doCreateArcticStreamWriter(tableLoader, submitEmptySnapshots, restoredCheckpointId); @@ -91,13 +91,13 @@ public static OneInputStreamOperatorTestHarness createArct } public static OneInputStreamOperatorTestHarness doCreateArcticStreamWriter( - ArcticTableLoader tableLoader, boolean submitEmptySnapshots, Long restoredCheckpointId) + AmoroTableLoader tableLoader, boolean submitEmptySnapshots, Long restoredCheckpointId) throws Exception { tableLoader.open(); MixedTable mixedTable = tableLoader.loadArcticTable(); mixedTable.properties().put(SUBMIT_EMPTY_SNAPSHOTS.key(), String.valueOf(submitEmptySnapshots)); - ArcticFileWriter streamWriter = + AmoroFileWriter streamWriter = FlinkSink.createFileWriter( mixedTable, null, @@ -129,11 +129,11 @@ public static TaskWriter createUnkeyedTaskWriter( @Test public void testInsertWrite() throws Exception { Assume.assumeTrue(isKeyedTable()); - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = createArcticStreamWriter(tableLoader)) { - ArcticFileWriter fileWriter = (ArcticFileWriter) testHarness.getOneInputOperator(); + AmoroFileWriter fileWriter = (AmoroFileWriter) testHarness.getOneInputOperator(); Assert.assertNotNull(fileWriter.getWriter()); // The first checkpoint testHarness.processElement(createRowData(1, "hello", "2020-10-11T10:10:11.0"), 1); @@ -166,7 +166,7 @@ public void testSnapshotMultipleTimes() throws Exception { long checkpointId = 1; long timestamp = 1; - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); try (OneInputStreamOperatorTestHarness testHarness = createArcticStreamWriter(tableLoader)) { testHarness.processElement(createRowData(1, "hello", "2020-10-11T10:10:11.0"), timestamp++); @@ -193,7 +193,7 @@ public void testSnapshotMultipleTimes() throws Exception { @Test public void testInsertWriteWithoutPk() throws Exception { Assume.assumeFalse(isKeyedTable()); - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = createArcticStreamWriter(tableLoader)) { @@ -224,7 +224,7 @@ public void testInsertWriteWithoutPk() throws Exception { @Test public void testDeleteWrite() throws Exception { Assume.assumeTrue(isKeyedTable()); - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = createArcticStreamWriter(tableLoader)) { @@ -262,7 +262,7 @@ public void testDeleteWrite() throws Exception { @Test public void testUpdateWrite() throws Exception { Assume.assumeTrue(isKeyedTable()); - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; try (OneInputStreamOperatorTestHarness testHarness = createArcticStreamWriter(tableLoader)) { @@ -301,7 +301,7 @@ public void testUpdateWrite() throws Exception { @Test public void testEmitEmptyResults() throws Exception { Assume.assumeTrue(isKeyedTable()); - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); long checkpointId = 1L; long excepted = submitEmptySnapshots ? 1 : 0; try (OneInputStreamOperatorTestHarness testHarness = diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java index 201c148f74..75179d3550 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticDoubleWriteStatus.java @@ -26,7 +26,7 @@ import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.table.MixedTable; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.iceberg.UpdateProperties; @@ -36,7 +36,7 @@ import java.time.Duration; public class TestAutomaticDoubleWriteStatus extends FlinkTestBase { - public ArcticTableLoader tableLoader; + public AmoroTableLoader tableLoader; public TestAutomaticDoubleWriteStatus() { super( @@ -46,7 +46,7 @@ public TestAutomaticDoubleWriteStatus() { @Test public void testTableProperties() { - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); tableLoader.open(); MixedTable mixedTable = tableLoader.loadArcticTable(); UpdateProperties up = mixedTable.updateProperties(); diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java index 0b363ccd1a..68cce86ab3 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestAutomaticLogWriter.java @@ -37,8 +37,8 @@ import org.apache.amoro.flink.metric.MetricsGenerator; import org.apache.amoro.flink.shuffle.LogRecordV1; import org.apache.amoro.flink.shuffle.ShuffleHelper; -import org.apache.amoro.flink.table.ArcticTableLoader; -import org.apache.amoro.flink.util.ArcticUtils; +import org.apache.amoro.flink.table.AmoroTableLoader; +import org.apache.amoro.flink.util.AmoroUtils; import org.apache.amoro.flink.util.DataUtil; import org.apache.amoro.flink.util.TestGlobalAggregateManager; import org.apache.amoro.flink.util.TestOneInputStreamOperatorIntern; @@ -94,7 +94,7 @@ @RunWith(Parameterized.class) public class TestAutomaticLogWriter extends FlinkTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestAutomaticLogWriter.class); - public ArcticTableLoader tableLoader; + public AmoroTableLoader tableLoader; public static final TestGlobalAggregateManager GLOBAL_AGGREGATE_MANGER = new TestGlobalAggregateManager(); @@ -131,7 +131,7 @@ public static void shutdown() throws Exception { @Before public void init() { - tableLoader = ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); + tableLoader = AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder); tableLoader.open(); } @@ -213,7 +213,7 @@ public void testHasCaughtUp() throws Exception { FlinkSink.forRowData(input) .context(Optional::of) .table(testKeyedTable) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkSchema(FLINK_SCHEMA) .producerConfig(getPropertiesByTopic(topic)) .topic(topic) @@ -391,10 +391,10 @@ private TestOneInputStreamOperatorIntern createProducer( Schema writeSchema = TypeUtil.reassignIds(FlinkSchemaUtil.convert(FLINK_SCHEMA), testKeyedTable.schema()); MetricsGenerator metricsGenerator = - ArcticUtils.getMetricsGenerator( + AmoroUtils.getMetricsGenerator( false, false, testKeyedTable, flinkSchemaRowType, writeSchema); - ArcticFileWriter streamWriter = + AmoroFileWriter streamWriter = FlinkSink.createFileWriter( testKeyedTable, null, @@ -402,8 +402,8 @@ private TestOneInputStreamOperatorIntern createProducer( (RowType) FLINK_SCHEMA.toRowDataType().getLogicalType(), tableLoader); - ArcticWriter arcticWriter = - new ArcticWriter<>(automaticLogWriter, streamWriter, metricsGenerator); + AmoroWriter arcticWriter = + new AmoroWriter<>(automaticLogWriter, streamWriter, metricsGenerator); TestOneInputStreamOperatorIntern harness = new TestOneInputStreamOperatorIntern<>( diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java index 8815db3da0..0f8329065d 100644 --- a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java +++ b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/write/TestFlinkSink.java @@ -23,7 +23,7 @@ import org.apache.amoro.TableTestHelper; import org.apache.amoro.catalog.BasicCatalogTestHelper; import org.apache.amoro.flink.FlinkTestBase; -import org.apache.amoro.flink.table.ArcticTableLoader; +import org.apache.amoro.flink.table.AmoroTableLoader; import org.apache.amoro.flink.util.DataUtil; import org.apache.amoro.io.MixedDataTestHelpers; import org.apache.amoro.table.KeyedTable; @@ -125,7 +125,7 @@ public void testKeyedSink() throws Exception { FlinkSink.forRowData(input) .context(Optional::of) .table(testKeyedTable) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkSchema(FLINK_SCHEMA) .build(); @@ -169,7 +169,7 @@ public void testUnkeyedSink() throws Exception { FlinkSink.forRowData(input) .context(Optional::of) .table(testTable) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkSchema(FLINK_SCHEMA) .build(); @@ -212,7 +212,7 @@ public void testUnkeyedOverwrite() throws Exception { FlinkSink.forRowData(input) .context(Optional::of) .table(testTable) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .flinkSchema(FLINK_SCHEMA) .build(); env.execute(); @@ -231,7 +231,7 @@ public void testUnkeyedOverwrite() throws Exception { FlinkSink.forRowData(overwrite) .context(Optional::of) .table(testTable) - .tableLoader(ArcticTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) + .tableLoader(AmoroTableLoader.of(TableTestHelper.TEST_TABLE_ID, catalogBuilder)) .overwrite(true) .flinkSchema(FLINK_SCHEMA) .build();