diff --git a/fe/fe-connector/fe-connector-hms-hive-shade/pom.xml b/fe/fe-connector/fe-connector-hms-hive-shade/pom.xml index d0391c238635cf..215bad3c367963 100644 --- a/fe/fe-connector/fe-connector-hms-hive-shade/pom.xml +++ b/fe/fe-connector/fe-connector-hms-hive-shade/pom.xml @@ -275,18 +275,26 @@ under the License. + the shade goal can start before this module's main jar is created and attached, + failing with "Failed to create shaded artifact, project main artifact does not + exist". Pin maven-jar-plugin's default-jar ahead of shade in the SAME phase (within + one phase the declaration order here decides) and force it to always (re)create the + jar, so the main artifact is guaranteed present before shade. + + That shared phase is process-classes, NOT package: this module carries no sources, + so its Hive metastore content exists only once shade has run. fe-connector-hms + compiles against that content, and run-fe-ut.sh builds with `mvn test`, which stops + before package — the reactor would then hand fe-connector-hms this module's (empty) + target/classes and every org.apache.hadoop.hive.* import would fail to resolve + (TeamCity FE UT 1009434). process-classes is the last phase still ahead of + test-compile. Mirrors fe-connector-paimon-hive-shade. --> org.apache.maven.plugins maven-jar-plugin default-jar - package + process-classes jar @@ -304,7 +312,7 @@ under the License. shade - package + process-classes + the shade goal can start before this module's main jar is created and attached, + failing with "Failed to create shaded artifact, project main artifact does not + exist". Pin maven-jar-plugin's default-jar ahead of shade in the SAME phase (within + one phase the declaration order here decides) and force it to always (re)create the + jar, so the main artifact is guaranteed present before shade — keeping parallel (-T) + builds reliable without falling back to single-threaded. + + That shared phase is process-classes, NOT package: this module carries no sources, + so its Hive metastore content exists only once shade has run. fe-connector-paimon + compiles against it (org.apache.hadoop.hive.conf.HiveConf), and run-fe-ut.sh builds + with `mvn test`, which stops before package — the reactor would then hand + fe-connector-paimon this module's (empty) target/classes and the HiveConf import + would fail to resolve. process-classes is the last phase still ahead of + test-compile. Mirrors fe-connector-hms-hive-shade. --> org.apache.maven.plugins maven-jar-plugin default-jar - package + process-classes jar @@ -246,7 +255,7 @@ under the License. shade - package + process-classes + + ${project.groupId} + fe-connector-hive + ${project.version} + test + + + ${project.groupId} + fe-connector-iceberg + ${project.version} + test + + + ${project.groupId} + fe-connector-paimon + ${project.version} + test + + + ${project.groupId} + fe-connector-es + ${project.version} + test + + + ${project.groupId} + fe-connector-jdbc + ${project.version} + test + + + ${project.groupId} + fe-connector-maxcompute + ${project.version} + test + + + ${project.groupId} + fe-connector-trino + ${project.version} + test + + + + ${project.groupId} + fe-connector-hudi + ${project.version} + test + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index 4b5c748b2ee6d7..aaf1ca9496eb68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -111,8 +111,26 @@ private static CatalogIf createCatalog(long catalogId, String name, String resou // takes to make its type usable here. Returns null when nothing claims it — including for a // sibling-only connector, whose type must never become a catalog (see ConnectorProvider // .isStandaloneCatalogType). - Connector spiConnector = ConnectorFactory.createStandaloneCatalogConnector( - catalogType, props, new DefaultConnectorContext(name, catalogId)); + Connector spiConnector; + try { + spiConnector = ConnectorFactory.createStandaloneCatalogConnector( + catalogType, props, new DefaultConnectorContext(name, catalogId)); + } catch (RuntimeException | Error e) { + if (!isReplay) { + // Creating a catalog interactively must still fail loud: the user is waiting for the error. + throw e; + } + // On replay we must not propagate. The edit-log replay fallback turns any exception from a + // replayed operation into System.exit(-1), so a connector constructor that rejects a property an + // older FE stored without validating — or a half-installed plugin throwing NoClassDefFoundError — + // would keep the whole FE from starting instead of making one catalog unusable. Fall through to + // the degraded registration below; the failure resurfaces at first access as a query error. + // (The image path never had this problem: it builds the connector lazily.) + LOG.warn("Connector for catalog type '{}' failed to build while replaying catalog '{}'. " + + "Registering it in degraded mode; accessing it will fail until the cause is fixed.", + catalogType, name, e); + spiConnector = null; + } if (spiConnector != null) { LOG.info("Created plugin-driven catalog '{}' via SPI connector for type '{}'", name, catalogType); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/Legacy413Fixtures.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/Legacy413Fixtures.java new file mode 100644 index 00000000000000..5f8f2c33d3b345 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/Legacy413Fixtures.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Access to the golden Doris 4.1.3 metadata fixtures under {@code src/test/resources/upgrade/413/}. + * + *

Those files were produced by real 4.1.3 bytecode, not written by hand; see the PROVENANCE.txt + * next to them for how to regenerate and for the five wire-format details that make hand-authored + * fixtures wrong. + */ +public final class Legacy413Fixtures { + + /** The lastUpdateTime the generator stamps on every catalog, so the bytes are deterministic. */ + public static final long FIXED_UPDATE_TIME = 1753000000000L; + + private static final String ROOT = "/upgrade/413/"; + + private Legacy413Fixtures() { + } + + /** The image's "datasource" module as a 4.1.3 FE would have written it. */ + public static CatalogMgr loadCatalogMgr() throws IOException { + return loadCatalogMgr("datasource.module.bin"); + } + + /** Same, for the variant generated with {@code Config.deploy_mode="cloud"}. */ + public static CatalogMgr loadCloudCatalogMgr() throws IOException { + return loadCatalogMgr("datasource.module.cloud.bin"); + } + + private static CatalogMgr loadCatalogMgr(String fileName) throws IOException { + try (InputStream in = open(fileName); + DataInputStream dis = new DataInputStream(in)) { + return CatalogMgr.read(dis); + } + } + + /** Raw bytes of one edit-log entry, in JournalEntity wire form (opCode short + Text-framed JSON). */ + public static byte[] journalEntry(String fileName) throws IOException { + try (InputStream in = open("editlog/" + fileName)) { + return readAll(in); + } + } + + /** Names of every edit-log fixture with the given op code, e.g. {@code 320}. */ + public static List journalEntryNames(int opCode) { + List names = new ArrayList<>(); + for (String name : listEditLogFixtures()) { + if (name.startsWith("op" + opCode + "-")) { + names.add(name); + } + } + if (names.isEmpty()) { + throw new IllegalStateException("no edit-log fixture for op " + opCode + + "; the resource index at " + ROOT + "editlog.index is stale"); + } + return names; + } + + private static List listEditLogFixtures() { + // Resource directories are not enumerable from a jar, and surefire may well run these classes off one. + // The generator writes a flat index next to the fixtures precisely so this does not need directory + // listing; keeping it in one place means a fixture added without regenerating the index fails loudly + // in journalEntryNames() rather than silently reducing coverage. + try (InputStream in = open("editlog.index")) { + String text = new String(readAll(in), StandardCharsets.UTF_8); + List names = new ArrayList<>(); + for (String line : text.split("\n")) { + String trimmed = line.trim(); + if (!trimmed.isEmpty() && !trimmed.startsWith("#")) { + names.add(trimmed); + } + } + return names; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * One of 4.1.3's own GSON label registries, read out of the generated {@code labels.*.txt}. + * + * @param which one of {@code ds}, {@code db}, {@code tbl} + * @return label to the fully-qualified name of the class 4.1.3 mapped it to, in file order + */ + public static Map legacyLabels(String which) throws IOException { + Map labels = new LinkedHashMap<>(); + try (InputStream in = open("labels." + which + ".txt")) { + String text = new String(readAll(in), StandardCharsets.UTF_8); + for (String line : text.split("\n")) { + if (line.trim().isEmpty() || line.startsWith("#")) { + continue; + } + String[] parts = line.split("\t"); + boolean isAbstract = parts.length > 2 && "ABSTRACT".equals(parts[2].trim()); + labels.put(parts[0], new LegacyLabel(parts[0], parts[1].trim(), isAbstract)); + } + } + return labels; + } + + /** One entry of a 4.1.3 GSON label registry. */ + public static final class LegacyLabel { + public final String label; + public final String legacyClassName; + /** + * Whether 4.1.3's class was abstract. GSON writes the runtime class's label, so an abstract + * class's label can never actually appear in an image -- it is registered but unreachable. + */ + public final boolean isAbstract; + + LegacyLabel(String label, String legacyClassName, boolean isAbstract) { + this.label = label; + this.legacyClassName = legacyClassName; + this.isAbstract = isAbstract; + } + + @Override + public String toString() { + return label + " -> " + legacyClassName + (isAbstract ? " (abstract)" : ""); + } + } + + private static InputStream open(String relative) throws IOException { + InputStream in = Legacy413Fixtures.class.getResourceAsStream(ROOT + relative); + if (in == null) { + throw new IOException("missing 4.1.3 fixture: " + ROOT + relative + + " -- see src/test/resources/upgrade/413/PROVENANCE.txt"); + } + return in; + } + + private static byte[] readAll(InputStream in) throws IOException { + byte[] buffer = new byte[8192]; + int size = 0; + int read; + while ((read = in.read(buffer, size, buffer.length - size)) > 0) { + size += read; + if (size == buffer.length) { + buffer = Arrays.copyOf(buffer, buffer.length * 2); + } + } + return Arrays.copyOf(buffer, size); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/Legacy413ImageMigrationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/Legacy413ImageMigrationTest.java new file mode 100644 index 00000000000000..79168d416e4a67 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/Legacy413ImageMigrationTest.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.Pair; +import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; +import org.apache.doris.datasource.log.InitCatalogLog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +/** + * Loading a real Doris 4.1.3 image's "datasource" module on this branch. + * + *

The fixture is not hand-written: it was emitted by 4.1.3 bytecode itself + * (see src/test/resources/upgrade/413/PROVENANCE.txt). That matters because 4.1.3's wire format has + * several details a hand-authored blob gets wrong -- HTML-escaped '=' and '&' inside jdbc_url, the + * "resource" key vanishing when null, and "taap" switching between a JSON object and an array of + * 2-tuples depending on whether the map is empty. + * + *

This class lives in {@code org.apache.doris.datasource} rather than beside the other upgrade tests + * because {@link ExternalCatalog#logType} is protected and there is no getter: asserting that the + * legacy logType really was rewritten to PLUGIN is the whole point of {@link #resourceBackedCatalogsRecoverTheirTypeFromLogType()}, + * and going through {@code getType()} instead would not observe it. + */ +public class Legacy413ImageMigrationTest { + + /** Every id in the fixture whose 4.1.3 class no longer exists and must therefore migrate. */ + private static final long[] MIGRATED_IDS = { + 10001L, // HMSExternalCatalog + 10002L, 10003L, 10004L, 10005L, 10006L, 10007L, 10008L, // Iceberg x 7 flavours + 10009L, // PaimonExternalCatalog + 10010L, 10011L, 10012L, 10013L, // Es / Jdbc / MaxCompute / Trino + 10020L, 10021L, 10022L, 10023L, // G2: resource-backed, no "type" prop + 10030L, 10031L, 10032L, 10033L, // G3: Paimon flavour carry-forward + 10034L, // G3: LakeSoul (retired, degraded) + }; + + @Test + public void everyLegacyExternalCatalogBecomesPluginDriven() throws Exception { + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + Assertions.assertEquals(24, mgr.getCatalogNum(), + "the 4.1.3 fixture holds the internal catalog plus 23 externals; a different count means the " + + "fixture was regenerated without updating this test"); + + for (long id : MIGRATED_IDS) { + CatalogIf catalog = mgr.getCatalog(id); + Assertions.assertNotNull(catalog, "catalog " + id + " disappeared while loading the 4.1.3 image"); + // assertSame, not instanceof: PluginDrivenMvccExternalTable extends PluginDrivenExternalTable and the + // same shape of silent downgrade is possible here if a future edit remaps a label to a subclass. + Assertions.assertSame(PluginDrivenExternalCatalog.class, catalog.getClass(), + "catalog " + id + " (" + catalog.getName() + ") must be served by the connector framework " + + "after the cutover, but loaded as " + catalog.getClass().getSimpleName()); + } + } + + @Test + public void catalogTypesTheEngineStillImplementsAreNotRoutedToPlugins() throws Exception { + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + // Control group. "doris" is a BUILTIN_CATALOG_TYPE served by the engine itself, so remapping it onto + // PluginDrivenExternalCatalog would be a silent regression that every other assertion here would miss. + CatalogIf remoteDoris = mgr.getCatalog(10015L); + Assertions.assertSame(RemoteDorisExternalCatalog.class, remoteDoris.getClass(), + "an engine-implemented catalog type must keep its own class across the cutover"); + + Assertions.assertSame(InternalCatalog.class, mgr.getCatalog(0L).getClass()); + } + + @Test + public void resourceBackedCatalogsRecoverTheirTypeFromLogType() throws Exception { + // THE migration test. These four catalogs carry NO "type" property: 4.1.3 never needed one because the + // concrete class carried the type. After the cutover the class is gone, so the type can only come from + // PluginDrivenExternalCatalog.gsonPostProcess backfilling it out of the persisted logType. + // + // Deleting that backfill leaves every G1 (type-bearing) assertion in this file green -- these four are + // the only ones that turn red. Do not "simplify" them away. + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + assertTypeRecovered(mgr, 10020L, "hms"); + assertTypeRecovered(mgr, 10021L, "es"); + assertTypeRecovered(mgr, 10022L, "jdbc"); + // The load-bearing case: TRINO_CONNECTOR is the only logType whose name().toLowerCase() + // ("trino_connector") is NOT the type string the connector answers to. A backfill that just lowercases + // the enum passes the other three and silently kills every migrated trino catalog. + assertTypeRecovered(mgr, 10023L, "trino-connector"); + } + + private void assertTypeRecovered(CatalogMgr mgr, long id, String expectedType) { + ExternalCatalog catalog = (ExternalCatalog) mgr.getCatalog(id); + + Assertions.assertEquals(expectedType, catalog.getType(), + "catalog " + id + " persisted no 'type' property, so its type must be recovered from logType"); + // Not just reported -- actually written back, because that map is what CatalogFactory and the connector + // providers dispatch on. A getType() that computes the answer on the fly would leave the catalog + // unusable the moment anything reads the property map instead. + Assertions.assertEquals(expectedType, + catalog.getCatalogProperty().getOrDefault(CatalogMgr.CATALOG_TYPE_PROP, ""), + "the recovered type must be persisted back into the property map, not just returned"); + Assertions.assertEquals(InitCatalogLog.Type.PLUGIN, catalog.logType, + "a migrated catalog must report the PLUGIN logType, otherwise buildDbForInit takes the legacy path"); + } + + @Test + public void catalogIdentityAndUserVisibleStateSurviveTheMigration() throws Exception { + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + ExternalCatalog hms = (ExternalCatalog) mgr.getCatalog(10001L); + Assertions.assertEquals("g1_hms", hms.getName()); + Assertions.assertEquals("my hive", hms.getComment(), "the user's COMMENT must survive the upgrade"); + // Asserted BEFORE anything initialises the catalog: makeSureInitialized() overwrites lastUpdateTime + // with System.currentTimeMillis(), so this can only be checked on a freshly deserialised object. + Assertions.assertEquals(Legacy413Fixtures.FIXED_UPDATE_TIME, hms.getLastUpdateTime()); + Assertions.assertEquals("thrift://hms-host:9083", + hms.getCatalogProperty().getOrDefault("hive.metastore.uris", "")); + + // taap uses a Pair key, which GSON writes as an array of 2-tuples via + // enableComplexMapKeySerialization. If that encoding ever changes, auto-analyze policies silently + // vanish on upgrade and nothing else in the suite would notice. + Map, String> policy = hms.tableAutoAnalyzePolicy; + Assertions.assertEquals(2, policy.size(), "auto-analyze policies must survive the upgrade"); + Assertions.assertEquals("enable", policy.get(Pair.of("db1", "tbl1"))); + Assertions.assertEquals("disable", policy.get(Pair.of("db2", "tbl2"))); + } + + @Test + public void jdbcUrlSurvivesHtmlEscapingRoundTrip() throws Exception { + // 4.1.3 persists jdbc_url with '=' and '&' HTML-escaped (= / &). This asserts the value is + // read back unescaped -- a fixture-fidelity canary as much as a product assertion: if someone replaces + // the golden bytes with a hand-typed blob, this is the first thing that breaks. + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + String jdbcUrl = ((ExternalCatalog) mgr.getCatalog(10011L)) + .getCatalogProperty().getOrDefault("jdbc_url", ""); + Assertions.assertTrue(jdbcUrl.startsWith("jdbc:mysql://mysql:3306/db?"), jdbcUrl); + Assertions.assertTrue(jdbcUrl.contains("useSSL=false"), "'=' must round-trip unescaped: " + jdbcUrl); + Assertions.assertTrue(jdbcUrl.contains("&serverTimezone=UTC"), "'&' must round-trip unescaped: " + jdbcUrl); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413CloudImageTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413CloudImageTest.java new file mode 100644 index 00000000000000..1d2def594c3af5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413CloudImageTest.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.upgrade; + +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.Legacy413Fixtures; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * The same 4.1.3 upgrade, in cloud (storage-compute separated) mode. + * + *

This is deliberately one small class rather than a second copy of the whole matrix. The only + * cloud-conditional branch anywhere on the catalog path is the tail of {@code GsonUtils}' + * {@code dsTypeAdapterFactory}, and it swaps exactly one label: {@code InternalCatalog} resolves to + * {@link CloudInternalCatalog}. Every external-catalog label, and every database and table label, is + * registered unconditionally. The fixture proves it empirically -- the generator was run twice on 4.1.3, + * once with {@code deploy_mode="cloud"}, and the two modules differ in exactly one entry (id 0). So a + * cloud copy of the external matrix would assert nothing the non-cloud one does not. + */ +public class Legacy413CloudImageTest { + + static { + // MUST be a static initialiser of this class, not @BeforeAll or @BeforeEach. + // GsonUtils builds its type-adapter factories in a static initialiser, and + // RuntimeTypeAdapterFactory.create() snapshots the label map, so flipping the mode after anything + // has touched GsonUtils is a SILENT no-op: the tests would still pass while asserting nothing about + // cloud mode. Safe because surefire runs fe-core with reuseForks=false, i.e. one fresh JVM per class. + Config.deploy_mode = "cloud"; + } + + @Test + public void cloudInternalCatalogSurvivesTheUpgrade() throws Exception { + CatalogMgr mgr = Legacy413Fixtures.loadCloudCatalogMgr(); + + Assertions.assertTrue(Config.isCloudMode(), "the deploy_mode latch must have been set before class init"); + Assertions.assertSame(CloudInternalCatalog.class, mgr.getCatalog(0L).getClass()); + } + + @Test + public void preCloudInternalCatalogLabelIsRemappedInCloudMode() throws Exception { + // The compat direction: metadata written before the cloud internal catalog got its own label carries + // "clazz":"InternalCatalog". In cloud mode that label must resolve to CloudInternalCatalog, otherwise + // a cloud FE upgraded from such an image loses its internal catalog entirely. + // The non-cloud fixture IS that metadata -- no synthetic JSON needed. + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + Assertions.assertSame(CloudInternalCatalog.class, mgr.getCatalog(0L).getClass(), + "in cloud mode the legacy 'InternalCatalog' label must resolve to CloudInternalCatalog"); + } + + @Test + public void externalCatalogMigrationIsIdenticalInCloudMode() throws Exception { + // Cheap regression guard, not a matrix: if a future change ever makes an external catalog's + // migration mode-dependent, this is what notices. + CatalogMgr mgr = Legacy413Fixtures.loadCloudCatalogMgr(); + + assertMigrated(mgr, 10001L, "hms"); + assertMigrated(mgr, 10003L, "iceberg"); + assertMigrated(mgr, 10009L, "paimon"); + // Resource-backed: the logType backfill must work the same way in cloud mode. + assertMigrated(mgr, 10023L, "trino-connector"); + } + + private void assertMigrated(CatalogMgr mgr, long id, String expectedType) { + CatalogIf catalog = mgr.getCatalog(id); + Assertions.assertSame(PluginDrivenExternalCatalog.class, catalog.getClass(), + "catalog " + id + " must migrate identically in cloud mode"); + Assertions.assertEquals(expectedType, ((ExternalCatalog) catalog).getType()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413JournalReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413JournalReplayTest.java new file mode 100644 index 00000000000000..c1ff50c8685adc --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413JournalReplayTest.java @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.upgrade; + +import org.apache.doris.common.io.Text; +import org.apache.doris.connector.ConnectorFactory; +import org.apache.doris.connector.ConnectorPluginManager; +import org.apache.doris.datasource.CatalogFactory; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.Legacy413Fixtures; +import org.apache.doris.datasource.log.CatalogLog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.persist.gson.GsonUtils; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Replaying a 4.1.3 FE's edit log on this branch. + * + *

This is the half of the upgrade that the image tests cannot reach. The two paths behave differently + * on purpose: loading an image only deserialises, and the connector is built lazily on first access, so a + * catalog whose plugin is missing or whose properties the connector rejects merely degrades. Replaying + * {@code OP_CREATE_CATALOG} instead builds the connector synchronously, on the thread that is starting the + * FE -- and {@code EditLog}'s fallback turns anything thrown there into {@code System.exit(-1)}. So the + * question "does an upgraded FE start" is decided here, not by the image tests. + * + *

The fixtures are the real bytes a 4.1.3 FE would have journalled: op code short followed by the + * Text-framed JSON of {@link CatalogLog}. Nothing about the payload class is on the wire, which is why a + * 4.1.3 entry and a current entry are byte-identical and only the replay behaviour differs. + */ +public class Legacy413JournalReplayTest { + + /** Catalog types the fixtures contain, i.e. the providers an upgraded FE must have installed. */ + private static final List FIXTURE_TYPES = com.google.common.collect.ImmutableList.of( + "hms", "iceberg", "paimon", "es", "jdbc", "max_compute", "trino-connector"); + + private RecordingConnectorProvider.Registry registry; + + @BeforeEach + public void setUp() { + // The plugin manager is a process-wide static shared with every other test class in this fork: + // start from a known-empty one rather than inheriting whatever ran before. + registry = RecordingConnectorProvider.installFor(FIXTURE_TYPES); + } + + @AfterEach + public void tearDown() { + ConnectorFactory.initPluginManager(new ConnectorPluginManager()); + } + + @Test + public void everyCreateCatalogEntryFrom413Replays() throws Exception { + // The whole point: a 4.1.3 master's journal, replayed by a follower running this branch, must not + // take the FE down. Asserting the returned object rather than "did not throw" matters -- a fix that + // swallowed the exception and returned null would satisfy assertDoesNotThrow and then NPE one frame + // up in CatalogMgr.replayCreateCatalog, landing back in System.exit(-1). + for (String fixture : Legacy413Fixtures.journalEntryNames(320)) { + CatalogLog log = readCatalogLog(fixture); + String type = log.getProps().get(CatalogMgr.CATALOG_TYPE_PROP); + + CatalogIf catalog = CatalogFactory.createFromLog(log); + + Assertions.assertNotNull(catalog, "replay of " + fixture + " produced no catalog"); + Assertions.assertTrue(catalog instanceof ExternalCatalog, + "replay of " + fixture + " produced " + catalog.getClass().getSimpleName()); + Assertions.assertEquals(log.getCatalogName(), catalog.getName()); + if (type != null && !"doris".equals(type) && !"lakesoul".equals(type)) { + Assertions.assertEquals(type, ((ExternalCatalog) catalog).getType(), + "replay of " + fixture + " must preserve the catalog type"); + } + } + } + + @Test + public void replayHandsTheConnectorExactlyThePropertiesThatWerePersisted() throws Exception { + // The closest a fe-core unit test can get to "the migrated catalog still works": we cannot reach a + // real metastore, but we can pin that the correct provider is chosen and that every property the + // 4.1.3 FE persisted arrives at it unchanged. A migration that silently drops or renames a property + // produces a catalog that loads fine and then cannot connect to anything. + CatalogLog log = readCatalogLog("op320-create-g1_hms.bin"); + + CatalogIf catalog = CatalogFactory.createFromLog(log); + ((ExternalCatalog) catalog).makeSureInitialized(); + + RecordingConnectorProvider hms = registry.get("hms"); + // Two constructions are expected, not one: CatalogFactory builds a connector with a lightweight + // context while the catalog does not exist yet, and initLocalObjectsImpl deliberately rebuilds it + // with the real engine context (execution authenticator for Kerberos/secured HMS) and closes the + // first. The one that matters is the last -- that is the connector the catalog actually runs on. + Assertions.assertFalse(hms.calls.isEmpty(), + "the provider claiming 'hms' must be asked to build the connector"); + Map delivered = hms.lastProperties(); + for (Map.Entry persisted : log.getProps().entrySet()) { + Assertions.assertEquals(persisted.getValue(), delivered.get(persisted.getKey()), + "property '" + persisted.getKey() + "' did not survive the journey from the 4.1.3 " + + "journal to the connector"); + } + Assertions.assertEquals("g1_hms", hms.calls.get(hms.calls.size() - 1).catalogName); + + // The superseded connector must be closed, otherwise every replayed catalog leaks a connection pool + // and a plugin classloader reference at FE startup -- invisible until an FE with many catalogs runs + // out of metaspace. + for (int i = 0; i < hms.created.size() - 1; i++) { + Assertions.assertTrue(hms.created.get(i).isClosed(), + "connector #" + i + " was replaced during initialisation but never closed"); + } + } + + @Test + public void connectorConstructorThatThrowsMustNotTakeTheFeDown() throws Exception { + // An upgraded FE replays properties that an older FE stored without the new connector's validation + // ever running. If the connector constructor rejects one of them, the exception surfaces on the + // replay thread, where EditLog turns it into System.exit(-1) -- the FE never starts, and the only + // way out is to hand-edit metadata. The image path has no such failure mode because it is lazy. + // + // We assert one frame below EditLog (asserting System.exit is not possible without killing the fork). + registry = RecordingConnectorProvider.installThrowingFor("hms"); + CatalogLog log = readCatalogLog("op320-create-g1_hms.bin"); + + CatalogIf catalog = CatalogFactory.createFromLog(log); + + Assertions.assertInstanceOf(PluginDrivenExternalCatalog.class, catalog, + "a connector that fails to construct during replay must leave a registered, degraded catalog " + + "behind, not propagate out of CatalogFactory.createFromLog"); + Assertions.assertEquals("hms", ((ExternalCatalog) catalog).getType()); + } + + @Test + public void replayOfATypeWithNoInstalledPluginDegradesInsteadOfFailing() throws Exception { + // The realistic upgrade accident: the new fe/lib is deployed but plugins/connector is not, so no + // provider claims any type. Every catalog must still register, so that only using one fails. + ConnectorFactory.initPluginManager(new ConnectorPluginManager()); + + for (String fixture : Legacy413Fixtures.journalEntryNames(320)) { + CatalogLog log = readCatalogLog(fixture); + CatalogIf catalog = CatalogFactory.createFromLog(log); + Assertions.assertNotNull(catalog, "replay of " + fixture + " must not fail when no plugin is installed"); + } + } + + private static CatalogLog readCatalogLog(String fixture) throws IOException { + byte[] bytes = Legacy413Fixtures.journalEntry(fixture); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes))) { + in.readShort(); // op code; the payload class is not on the wire + return GsonUtils.GSON.fromJson(Text.readString(in), CatalogLog.class); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413LabelRegistryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413LabelRegistryTest.java new file mode 100644 index 00000000000000..ede333db8ce0ef --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413LabelRegistryTest.java @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.upgrade; + +import org.apache.doris.datasource.Legacy413Fixtures; +import org.apache.doris.datasource.Legacy413Fixtures.LegacyLabel; +import org.apache.doris.datasource.mvcc.PluginDrivenMvccExternalTable; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalDatabase; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.persist.gson.RuntimeTypeAdapterFactory; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Every GSON {@code clazz} label a 4.1.3 FE could have written must still resolve on this branch. + * + *

Those labels are bare string literals in {@code GsonUtils}: nothing in the compiler or in any other + * test notices if one is deleted or mistyped, and the consequence is not a degraded catalog but an FE that + * cannot start at all, because an unregistered label makes the whole image fail to parse. + * + *

The expectation is not hand-maintained. {@code labels.*.txt} is dumped reflectively out of 4.1.3's own + * registries by the fixture generator, so this test compares one machine-read registry against another. + * That matters: the obvious alternative -- scraping GsonUtils with a regex -- silently drops + * {@code IcebergS3TablesExternalCatalog}, whose class name contains a digit. + */ +public class Legacy413LabelRegistryTest { + + /** + * The one thing that genuinely is a decision rather than a mechanical consequence: which legacy table + * labels land on the MVCC-capable table class. Getting this wrong is silent -- the catalog still loads, + * queries still plan, and only snapshot/time-travel/MTMV freshness quietly misbehaves. + */ + private static final Map> EXPECTED_TABLE_TARGETS = ImmutableMap.>builder() + .put("HMSExternalTable", PluginDrivenMvccExternalTable.class) + .put("IcebergExternalTable", PluginDrivenMvccExternalTable.class) + .put("PaimonExternalTable", PluginDrivenMvccExternalTable.class) + .put("EsExternalTable", PluginDrivenExternalTable.class) + .put("JdbcExternalTable", PluginDrivenExternalTable.class) + .put("TrinoConnectorExternalTable", PluginDrivenExternalTable.class) + .put("MaxComputeExternalTable", PluginDrivenExternalTable.class) + .put("LakeSoulExternalTable", PluginDrivenExternalTable.class) + .build(); + + @Test + public void everyCatalogLabelWrittenBy413StillResolves() throws Exception { + assertRegistryCoversLegacy("ds", "dsTypeAdapterFactory", PluginDrivenExternalCatalog.class); + } + + @Test + public void everyDatabaseLabelWrittenBy413StillResolves() throws Exception { + assertRegistryCoversLegacy("db", "dbTypeAdapterFactory", PluginDrivenExternalDatabase.class); + } + + @Test + public void everyTableLabelWrittenBy413StillResolves() throws Exception { + assertRegistryCoversLegacy("tbl", "tblTypeAdapterFactory", PluginDrivenExternalTable.class); + } + + /** + * For each label 4.1.3 registered: this branch must register it too, and it must resolve to the same + * class when that class still exists, or to a plugin-driven replacement when it was deleted by the + * cutover. Encoding the rule rather than a frozen list means a newly deleted connector class is caught + * without anyone remembering to update a table here. + */ + private void assertRegistryCoversLegacy(String which, String factoryField, Class pluginDrivenBase) + throws Exception { + Map legacy = Legacy413Fixtures.legacyLabels(which); + Map> current = currentRegistry(factoryField); + + List problems = new ArrayList<>(); + for (LegacyLabel label : legacy.values()) { + Class target = current.get(label.label); + if (target == null) { + problems.add("label '" + label.label + "' was written by 4.1.3 but is not registered on this " + + "branch: any image containing it fails to parse and the FE cannot start"); + continue; + } + Class legacyClass = classOrNull(label.legacyClassName); + if (legacyClass != null) { + problems.add(checkSurvivingClass(label, target, legacyClass)); + } else { + problems.add(checkMigratedClass(label, target, pluginDrivenBase)); + } + } + problems.removeIf(java.util.Objects::isNull); + Assertions.assertTrue(problems.isEmpty(), String.join("\n", problems)); + + Assertions.assertFalse(legacy.isEmpty(), "labels." + which + ".txt is empty; the fixture is broken"); + } + + private String checkSurvivingClass(LegacyLabel label, Class target, Class legacyClass) { + // The class survived the cutover, so the label must still point at it -- silently repointing a + // surviving class at a plugin-driven one would change behaviour for catalogs the engine still owns. + if (!target.equals(legacyClass)) { + return "label '" + label.label + "' pointed at " + legacyClass.getName() + + " in 4.1.3 and that class still exists, but this branch resolves it to " + + target.getName(); + } + return null; + } + + private String checkMigratedClass(LegacyLabel label, Class target, Class pluginDrivenBase) { + if (!pluginDrivenBase.isAssignableFrom(target)) { + return "label '" + label.label + "' lost its 4.1.3 class (" + label.legacyClassName + + ") to the connector cutover, so it must resolve to a " + pluginDrivenBase.getSimpleName() + + ", but resolves to " + target.getName(); + } + Class expected = EXPECTED_TABLE_TARGETS.get(label.label); + if (expected != null && !expected.equals(target)) { + // assertSame-style, deliberately: PluginDrivenMvccExternalTable extends PluginDrivenExternalTable, + // so an isAssignableFrom check alone accepts a silent downgrade to the non-MVCC variant. + return "label '" + label.label + "' must resolve to " + expected.getSimpleName() + + " (MVCC capability is a decision, not a default), but resolves to " + target.getSimpleName(); + } + return null; + } + + @Test + public void abstractLegacyClassesAreRecordedAsUnreachable() throws Exception { + // IcebergExternalCatalog is registered by 4.1.3 but abstract, so GSON writes the concrete flavour's + // label instead and this one can never appear in an image. Recording that in the fixture keeps a + // future reader from "fixing" its absence from the image fixture. + Map legacy = Legacy413Fixtures.legacyLabels("ds"); + LegacyLabel iceberg = legacy.get("IcebergExternalCatalog"); + Assertions.assertNotNull(iceberg, "4.1.3 registered IcebergExternalCatalog"); + Assertions.assertTrue(iceberg.isAbstract, + "IcebergExternalCatalog was abstract in 4.1.3; if it is now concrete the image fixture must " + + "gain an entry for it"); + } + + private static Map> currentRegistry(String factoryField) throws Exception { + Field field = GsonUtils.class.getDeclaredField(factoryField); + field.setAccessible(true); + RuntimeTypeAdapterFactory factory = (RuntimeTypeAdapterFactory) field.get(null); + Field labelToSubtype = RuntimeTypeAdapterFactory.class.getDeclaredField("labelToSubtype"); + labelToSubtype.setAccessible(true); + @SuppressWarnings("unchecked") + Map> registry = (Map>) labelToSubtype.get(factory); + return registry; + } + + private static Class classOrNull(String name) { + try { + return Class.forName(name); + } catch (ClassNotFoundException | NoClassDefFoundError e) { + return null; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413ProviderTypeContractTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413ProviderTypeContractTest.java new file mode 100644 index 00000000000000..145386f9906212 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/Legacy413ProviderTypeContractTest.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.upgrade; + +import org.apache.doris.connector.ConnectorFactory; +import org.apache.doris.connector.ConnectorPluginManager; +import org.apache.doris.connector.spi.ConnectorProvider; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.Legacy413Fixtures; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The catalog type a migrated 4.1.3 catalog recovers must be a type a REAL shipped connector answers to. + * + *

This is the one upgrade property that no other test here can express. The recovery side lives in + * {@code PluginDrivenExternalCatalog.legacyLogTypeToCatalogType} as string literals; the answering side + * lives in each connector's {@code ConnectorProvider.getType()}, in a different maven module that fe-core + * does not depend on at compile or runtime scope. Every other test in this repository asserts one side + * against a literal copied out of the file under test, so renaming + * {@code TrinoConnectorProvider.getType()} to {@code "trino"} leaves the whole suite green while every + * migrated trino catalog is permanently unusable. The connector modules are on the test classpath (test + * scope only, see fe-core/pom.xml) purely so this class can compare the two sides against each other. + * + *

Production discovers providers from the {@code plugins/connector/} directory rather than from the + * classpath, so the set found here is "what this build ships", which is exactly the set this assertion + * needs. + */ +public class Legacy413ProviderTypeContractTest { + + /** + * The fixture ids whose properties carry no {@code type}: their catalog type exists only because the + * migration recovered it from logType. These are the ones where a mismatch is fatal and invisible. + */ + private static final long[] RESOURCE_BACKED_IDS = {10020L, 10021L, 10022L, 10023L}; + + /** Ids whose type came straight off the wire; included so a provider rename is caught for them too. */ + private static final long[] TYPE_BEARING_IDS = { + 10001L, 10002L, 10003L, 10004L, 10005L, 10006L, 10007L, 10008L, + 10009L, 10010L, 10011L, 10012L, 10013L, + }; + + @BeforeEach + public void setUp() { + ConnectorPluginManager manager = new ConnectorPluginManager(); + // ServiceLoader over the test classpath, i.e. the providers this build actually ships. + manager.loadBuiltins(); + ConnectorFactory.initPluginManager(manager); + } + + @AfterEach + public void tearDown() { + ConnectorFactory.initPluginManager(new ConnectorPluginManager()); + } + + @Test + public void theBuildActuallyShipsProviders() { + // Guard against the whole class passing vacuously: if the test-scope dependencies were dropped, + // ServiceLoader finds nothing, findProvider returns empty for everything, and the assertions below + // would have nothing to disagree with. + Assertions.assertFalse(ConnectorFactory.getStandaloneCatalogTypes().isEmpty(), + "no ConnectorProvider is on the test classpath: the fe-connector-* test-scope dependencies " + + "in fe-core/pom.xml were removed, and every assertion in this class is now vacuous"); + } + + @Test + public void everyMigratedCatalogTypeIsClaimedByAShippedProvider() throws Exception { + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + + List unclaimed = new ArrayList<>(); + for (long id : concat(RESOURCE_BACKED_IDS, TYPE_BEARING_IDS)) { + ExternalCatalog catalog = (ExternalCatalog) mgr.getCatalog(id); + String type = catalog.getType(); + Map props = catalog.getCatalogProperty().getProperties(); + Optional provider = ConnectorFactory.findProvider(type, props); + if (!provider.isPresent()) { + unclaimed.add("catalog " + id + " ('" + catalog.getName() + "') migrated to type '" + type + + "', which no shipped connector provider answers to; installed types: " + + ConnectorFactory.getStandaloneCatalogTypes()); + } + } + Assertions.assertTrue(unclaimed.isEmpty(), String.join("\n", unclaimed)); + } + + @Test + public void theHyphenatedTrinoTypeSurvivesBothSidesOfTheContract() throws Exception { + // Called out separately because it is the single most fragile pairing: the recovery side has to + // special-case TRINO_CONNECTOR (name().toLowerCase() gives "trino_connector"), and the provider side + // has to spell it the same way. Every other type happens to agree by accident of naming. + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + ExternalCatalog trino = (ExternalCatalog) mgr.getCatalog(10023L); + + Assertions.assertEquals("trino-connector", trino.getType()); + Assertions.assertTrue( + ConnectorFactory.findProvider(trino.getType(), trino.getCatalogProperty().getProperties()) + .isPresent(), + "the type recovered from logType TRINO_CONNECTOR must be the exact string the shipped " + + "trino provider answers to"); + } + + @Test + public void siblingOnlyConnectorNeverBecomesACatalogType() { + // hudi ships a provider but is sibling-only: it serves tables parasitic on another connector's + // metastore. If it ever started claiming standalone catalogs, "CREATE CATALOG ... type=hudi" would + // build a catalog with no semantics, and a migrated hms catalog could be routed to it. + Assertions.assertFalse(ConnectorFactory.getStandaloneCatalogTypes().contains("hudi"), + "hudi is sibling-only and must never be advertised as a creatable catalog type"); + } + + @Test + public void engineOwnedTypesAreNotClaimedByAnyPlugin() throws Exception { + // The control from the image side, restated against real providers: "doris" stays with the engine. + CatalogMgr mgr = Legacy413Fixtures.loadCatalogMgr(); + CatalogIf remoteDoris = mgr.getCatalog(10015L); + + Assertions.assertFalse(ConnectorFactory.getStandaloneCatalogTypes().contains("doris"), + "'doris' is implemented by the engine; a plugin claiming it would silently take over " + + remoteDoris.getClass().getSimpleName() + " catalogs on upgrade"); + } + + private static long[] concat(long[] a, long[] b) { + long[] out = new long[a.length + b.length]; + System.arraycopy(a, 0, out, 0, a.length); + System.arraycopy(b, 0, out, a.length, b.length); + return out; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/RecordingConnectorProvider.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/RecordingConnectorProvider.java new file mode 100644 index 00000000000000..9eff7254a4b450 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/upgrade/RecordingConnectorProvider.java @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.upgrade; + +import org.apache.doris.connector.ConnectorFactory; +import org.apache.doris.connector.ConnectorPluginManager; +import org.apache.doris.connector.api.Connector; +import org.apache.doris.connector.api.ConnectorMetadata; +import org.apache.doris.connector.api.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorProvider; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * A connector provider that records what it was asked to build. + * + *

fe-core depends only on {@code fe-connector-api} and {@code fe-connector-spi}, so no real connector is + * on the unit-test classpath and every catalog would otherwise land in the degraded arm of + * {@code CatalogFactory}. Registering a recorder instead buys the strongest statement a fe-core test can + * make about a migrated catalog: the right provider was chosen, and the properties that reach it are + * exactly the ones the old FE persisted. A real connector could not be used for this even if it were on + * the classpath -- there is no way to observe the map it received. + */ +public class RecordingConnectorProvider implements ConnectorProvider { + + /** One {@code create()} invocation. */ + public static final class Call { + public final Map properties; + public final String catalogName; + public final long catalogId; + + Call(Map properties, ConnectorContext context) { + // Defensive snapshot: the caller may keep mutating the map it handed us. + this.properties = new TreeMap<>(properties); + this.catalogName = context.getCatalogName(); + this.catalogId = context.getCatalogId(); + } + } + + /** The providers installed for one test, by catalog type. */ + public static final class Registry { + private final Map byType = new HashMap<>(); + + public RecordingConnectorProvider get(String type) { + RecordingConnectorProvider provider = byType.get(type); + if (provider == null) { + throw new IllegalArgumentException("no recording provider installed for type '" + type + + "'; installed: " + byType.keySet()); + } + return provider; + } + } + + public final List calls = new ArrayList<>(); + + private final String type; + private final boolean failOnCreate; + + private RecordingConnectorProvider(String type, boolean failOnCreate) { + this.type = type; + this.failOnCreate = failOnCreate; + } + + /** Installs one recording provider per type, replacing whatever the previous test class registered. */ + public static Registry installFor(List types) { + Registry registry = new Registry(); + ConnectorPluginManager manager = new ConnectorPluginManager(); + for (String type : types) { + RecordingConnectorProvider provider = new RecordingConnectorProvider(type, false); + registry.byType.put(type, provider); + manager.registerProvider(provider); + } + ConnectorFactory.initPluginManager(manager); + return registry; + } + + /** + * Installs a provider whose {@code create()} throws, standing in for a connector that rejects a + * property an older FE stored without validating, or a half-installed plugin. + */ + public static Registry installThrowingFor(String type) { + Registry registry = new Registry(); + ConnectorPluginManager manager = new ConnectorPluginManager(); + RecordingConnectorProvider provider = new RecordingConnectorProvider(type, true); + registry.byType.put(type, provider); + manager.registerProvider(provider); + ConnectorFactory.initPluginManager(manager); + return registry; + } + + @Override + public String getType() { + return type; + } + + @Override + public boolean isStandaloneCatalogType() { + return true; + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + calls.add(new Call(properties, context)); + if (failOnCreate) { + throw new IllegalArgumentException( + "simulated connector rejection of a property persisted by an older FE"); + } + RecordingConnector connector = new RecordingConnector(); + created.add(connector); + return connector; + } + + /** Every connector this provider handed out, in creation order. */ + public final List created = new ArrayList<>(); + + /** The properties handed to the connector that actually ends up serving the catalog. */ + public Map lastProperties() { + if (calls.isEmpty()) { + throw new IllegalStateException("the provider was never asked to build a connector"); + } + return calls.get(calls.size() - 1).properties; + } + + /** A connector that remembers whether it was closed, so leaked instances are observable. */ + public static final class RecordingConnector implements Connector { + private boolean closed; + + public boolean isClosed() { + return closed; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return null; + } + + @Override + public void close() { + closed = true; + } + } +} diff --git a/fe/fe-core/src/test/resources/upgrade/413/PROVENANCE.txt b/fe/fe-core/src/test/resources/upgrade/413/PROVENANCE.txt new file mode 100644 index 00000000000000..f1b80622a03152 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/413/PROVENANCE.txt @@ -0,0 +1,108 @@ +Golden FE metadata fixtures for the 4.1.3 -> catalog-SPI upgrade compatibility tests. + +WHAT THIS IS +============ +Every .bin / .json file in this directory was produced by REAL Apache Doris 4.1.3 bytecode. +Nothing here was hand-written. That is the point: a hand-authored fixture only encodes what +someone believed 4.1.3 writes, and 4.1.3's actual wire format has at least five details that +are easy to get wrong (see TRAPS below). + +PROVENANCE +========== + source tag : 4.1.3 (annotated tag; commit 7126cf65d96ebc43fce0906f51e92c1a2ccf24a6) + gson : 2.10.1 (identical on 4.1.3 and on the branch under test) + generator : generator/Gen413Fixtures.java (committed here verbatim) + generated on : 2026-07-29 + sha256 datasource.module.bin : cda84219f44d46325f38103f7d34655636b79723c4f7cbb2488c2230f6d06bf6 + sha256 datasource.module.cloud.bin : b1d1f55e331d691d247847d31e9069a0afd08fe3e6556cfb405a1965b301560e + +HOW TO REGENERATE +================= + git worktree add --detach /path/to/doris-413 4.1.3^{commit} + cp generator/Gen413Fixtures.java /path/to/doris-413/fe/fe-core/src/main/java/org/apache/doris/tools/ + cd /path/to/doris-413 + DORIS_HOME=$PWD DORIS_THIRDPARTY=/path/to/doris/thirdparty bash generated-source.sh + mvn -o -f /path/to/doris-413/fe/pom.xml -pl fe-core -am -DskipTests \ + -Dcheckstyle.skip=true -Dlicense.skip=true -Drat.skip=true compile + mvn -o -f /path/to/doris-413/fe/pom.xml -pl fe-core dependency:build-classpath \ + -Dmdep.outputFile=/tmp/cp413.txt -Dmdep.includeScope=runtime + # PREPEND the 4.1.3 worktree's own target/classes dirs, otherwise the ~/.m2 jars + # (fe-common 1.2-SNAPSHOT etc.) silently supply the CURRENT branch's classes: + LOCAL=$(find /path/to/doris-413/fe -maxdepth 3 -type d -name classes -path '*/target/*' | tr '\n' ':') + java -cp "${LOCAL}$(cat /tmp/cp413.txt)" org.apache.doris.tools.Gen413Fixtures [cloud] + + Output is byte-deterministic across JVM runs (verified: two runs produce identical sha256). + + !! NEVER run `mvn install` in the 4.1.3 worktree. Both trees resolve ${revision}=1.2-SNAPSHOT, + so 4.1.3's jars would silently overwrite the current branch's artifacts in ~/.m2. + !! `-am` is mandatory on every mvn invocation there; a single-module build fails on ${revision}. + +CONTENTS +======== + datasource.module.bin the image's "datasource" module, framed exactly as MetaWriter writes it + (Text.writeString = big-endian int32 length + UTF-8). 24 catalogs. + datasource.module.cloud.bin the same generator run with Config.deploy_mode="cloud". + Differs from the non-cloud file in EXACTLY ONE entry: id 0 is + {"clazz":"CloudInternalCatalog"} instead of {"clazz":"InternalCatalog"}. + Every external catalog entry is byte-identical -> external-catalog + migration is provably cloud-mode independent. + catalogMgr[.cloud].json the same JSON unframed, for human diffing. Not read by the tests. + labels.{ds,db,tbl}.txt 4.1.3's own GSON label registries, read reflectively out of + GsonUtils' RuntimeTypeAdapterFactory instances. Machine-generated on + purpose: a regex scrape drops entries whose class name contains a digit + (IcebergS3TablesExternalCatalog). + editlog/op-.bin one BDBJE journal entry each, in JournalEntity wire form: + writeShort(opCode) followed by Text.writeString(GSON.toJson(payload)). + generator/ the generator source, so the fixtures are reproducible. + +FIXTURE FAMILIES (why each exists) +================================== + G1 ids 10001-10015 built through 4.1.3's own CatalogFactory.createFromLog, props DO carry "type". + This is what a catalog CREATED ON 4.1.3 looks like. + 10015 (RemoteDorisExternalCatalog) is a CONTROL: the engine implements "doris" + itself, so it must NOT be remapped onto PluginDrivenExternalCatalog. + G2 ids 10020-10023 resource-backed shape: catalogProperty.resource is set and props carry NO "type" + key. 4.1.3 did not need "type" because the concrete class carried it. + *** THIS FAMILY IS THE ONLY ONE THAT CAN FAIL WHEN THE MIGRATION IS REMOVED. *** + With "type" present (G1), deleting PluginDrivenExternalCatalog.gsonPostProcess' + backfill entirely leaves every assertion green, because getType() just reads the + property that was already on the wire. Do not add fixtures without a G2 twin. + 10023 (trino) is load-bearing: TRINO_CONNECTOR is the only logType whose + name().toLowerCase() ("trino_connector") differs from the catalog type string + the connector answers to ("trino-connector"). + G3 ids 10030-10034 labels that exist in 4.1.3's registry and can appear in a 4.1 image, but that + 4.1.3's own factories can no longer produce: the paimon factory always returns + the base PaimonExternalCatalog, and lakesoul creation is rejected outright. + They are carry-forward from <= 4.0 metadata. Still real 4.1.3 bytecode -- + only the construction route differs (direct constructor + setDefaultPropsIfMissing, + which is literally what 4.1.3's CatalogFactory switch does after picking a class). + +LABELS DELIBERATELY ABSENT FROM THE FIXTURE +=========================================== + IcebergExternalCatalog registered by 4.1.3 but ABSTRACT, so GSON can never emit that label. + (labels.ds.txt marks it.) + TestExternalCatalog needs a unit-test-only provider class; cannot occur in a production image. + CloudInternalCatalog present in datasource.module.cloud.bin instead. + +TRAPS THAT MAKE HAND-WRITTEN FIXTURES WRONG (all observed in these bytes) +======================================================================== + 1. HTML escaping is ON. jdbc_url persists as + "jdbc:mysql://mysql:3306/db?useSSL=false&serverTimezone=UTC..." + -- '=' and '&' are escaped. + 2. serializeNulls is OFF. catalogProperty.resource is absent entirely when null, not "resource":null. + 3. taap (tableAutoAnalyzePolicy) is polymorphic under enableComplexMapKeySerialization: + empty map -> "taap":{} (a JSON object) + non-empty map -> "taap":[[{"first":"db1","second":"tbl1"},"enable"],...] (array of 2-tuples) + 4. "clazz" is written FIRST, then ExternalCatalog's 8 fields in declaration order: + id, name, logType, catalogProperty, initialized, lastUpdateTime, taap, comment. + 5. What is persisted is NOT what the user typed. setDefaultPropsIfMissing injects use_meta_cache / + enable.mapping.varbinary / enable.mapping.timestamp_tz into every catalog, HMS adds + ipc.client.fallback-to-simple-auth-allowed, and JDBC rewrites jdbc_url (appending + yearIsDateType=false&tinyInt1isBit=false&...). + +ONE-WAY DOOR +============ +Downgrade is out of scope BY DECISION, and it is not merely untested -- it is impossible. +The branch writes "clazz":"PluginDrivenExternalCatalog", which no 4.1.3 FE can parse +(no such label, and CatalogIf has no default subtype), so once an upgraded FE checkpoints, +rolling the binary back to 4.1.3 cannot load its own image. diff --git a/fe/fe-core/src/test/resources/upgrade/413/catalogMgr.cloud.json b/fe/fe-core/src/test/resources/upgrade/413/catalogMgr.cloud.json new file mode 100644 index 00000000000000..7f73496d50c664 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/413/catalogMgr.cloud.json @@ -0,0 +1 @@ +{"idToCatalog":{"0":{"clazz":"CloudInternalCatalog"},"10001":{"clazz":"HMSExternalCatalog","id":10001,"name":"g1_hms","logType":"HMS","catalogProperty":{"properties":{"type":"hms","hive.metastore.uris":"thrift://hms-host:9083","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false","ipc.client.fallback-to-simple-auth-allowed":"true"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":[[{"first":"db1","second":"tbl1"},"enable"],[{"first":"db2","second":"tbl2"},"disable"]],"comment":"my hive"},"10002":{"clazz":"IcebergHMSExternalCatalog","id":10002,"name":"g1_ice_hms","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"hms","hive.metastore.uris":"thrift://hms-host:9083","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10003":{"clazz":"IcebergRestExternalCatalog","id":10003,"name":"g1_ice_rest","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"rest","uri":"http://rest:8181","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10004":{"clazz":"IcebergGlueExternalCatalog","id":10004,"name":"g1_ice_glue","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"glue","glue.region":"us-east-1","warehouse":"s3://b/w","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10005":{"clazz":"IcebergDLFExternalCatalog","id":10005,"name":"g1_ice_dlf","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"dlf","dlf.region":"cn-beijing","hive.metastore.type":"dlf","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10006":{"clazz":"IcebergHadoopExternalCatalog","id":10006,"name":"g1_ice_hadoop","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"hadoop","warehouse":"hdfs://nn:8020/wh","fs.defaultFS":"hdfs://nn:8020","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10007":{"clazz":"IcebergJdbcExternalCatalog","id":10007,"name":"g1_ice_jdbc","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"jdbc","uri":"jdbc:postgresql://pg:5432/ice","warehouse":"s3://b/w","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10008":{"clazz":"IcebergS3TablesExternalCatalog","id":10008,"name":"g1_ice_s3tables","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"s3tables","warehouse":"arn:aws:s3tables:us-east-1:1:bucket/b","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10009":{"clazz":"PaimonExternalCatalog","id":10009,"name":"g1_paimon","logType":"PAIMON","catalogProperty":{"properties":{"type":"paimon","warehouse":"file:/tmp/paimon_wh","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":"paimon fs"},"10010":{"clazz":"EsExternalCatalog","id":10010,"name":"g1_es","logType":"ES","catalogProperty":{"properties":{"enable.mapping.varbinary":"false","use_meta_cache":"true","hosts":"http://es:9200","enable.mapping.timestamp_tz":"false","type":"es"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10011":{"clazz":"JdbcExternalCatalog","id":10011,"name":"g1_jdbc","logType":"JDBC","catalogProperty":{"properties":{"password":"pwd","enable.mapping.varbinary":"false","driver_class":"com.mysql.cj.jdbc.Driver","use_meta_cache":"true","enable.mapping.timestamp_tz":"false","driver_url":"mysql-connector-j-8.3.0.jar","type":"jdbc","user":"root","jdbc_url":"jdbc:mysql://mysql:3306/db?useSSL\u003dfalse\u0026serverTimezone\u003dUTC\u0026yearIsDateType\u003dfalse\u0026tinyInt1isBit\u003dfalse\u0026useUnicode\u003dtrue\u0026rewriteBatchedStatements\u003dtrue\u0026characterEncoding\u003dutf-8"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10012":{"clazz":"MaxComputeExternalCatalog","id":10012,"name":"g1_maxcompute","logType":"MAX_COMPUTE","catalogProperty":{"properties":{"type":"max_compute","mc.default.project":"p","mc.access_key":"ak","mc.secret_key":"sk","mc.endpoint":"http://service.cn.maxcompute.aliyun.com/api","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10013":{"clazz":"TrinoConnectorExternalCatalog","id":10013,"name":"g1_trino","logType":"TRINO_CONNECTOR","catalogProperty":{"properties":{"type":"trino-connector","trino.connector.name":"tpch","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10015":{"clazz":"RemoteDorisExternalCatalog","id":10015,"name":"g1_remote_doris","logType":"REMOTE_DORIS","catalogProperty":{"properties":{"type":"doris","jdbc.jdbc_url":"jdbc:mysql://doris-fe:9030/db","jdbc.user":"root","jdbc.password":"","jdbc.driver_url":"mysql-connector-j-8.3.0.jar","jdbc.driver_class":"com.mysql.cj.jdbc.Driver","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10020":{"clazz":"HMSExternalCatalog","id":10020,"name":"g2_hms_res","logType":"HMS","catalogProperty":{"resource":"hms_res","properties":{"hive.metastore.uris":"thrift://hms-host:9083","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false","ipc.client.fallback-to-simple-auth-allowed":"true"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":"resource backed hms"},"10021":{"clazz":"EsExternalCatalog","id":10021,"name":"g2_es_res","logType":"ES","catalogProperty":{"resource":"es_res","properties":{"enable.mapping.varbinary":"false","use_meta_cache":"true","hosts":"http://es:9200","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10022":{"clazz":"JdbcExternalCatalog","id":10022,"name":"g2_jdbc_res","logType":"JDBC","catalogProperty":{"resource":"jdbc_res","properties":{"password":"pwd","enable.mapping.varbinary":"false","driver_class":"com.mysql.cj.jdbc.Driver","use_meta_cache":"true","enable.mapping.timestamp_tz":"false","driver_url":"mysql-connector-j-8.3.0.jar","user":"root","jdbc_url":"jdbc:mysql://mysql:3306/db?useSSL\u003dfalse\u0026serverTimezone\u003dUTC\u0026yearIsDateType\u003dfalse\u0026tinyInt1isBit\u003dfalse\u0026useUnicode\u003dtrue\u0026rewriteBatchedStatements\u003dtrue\u0026characterEncoding\u003dutf-8"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10023":{"clazz":"TrinoConnectorExternalCatalog","id":10023,"name":"g2_trino_res","logType":"TRINO_CONNECTOR","catalogProperty":{"resource":"trino_res","properties":{"trino.connector.name":"tpch","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10030":{"clazz":"PaimonHMSExternalCatalog","id":10030,"name":"g3_paimon_hms","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"hms","hive.metastore.uris":"thrift://hms-host:9083","warehouse":"hdfs://nn:8020/wh","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10031":{"clazz":"PaimonFileExternalCatalog","id":10031,"name":"g3_paimon_file","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"filesystem","warehouse":"file:/tmp/paimon_wh","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10032":{"clazz":"PaimonRestExternalCatalog","id":10032,"name":"g3_paimon_rest","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"rest","uri":"http://paimon-rest:8080","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10033":{"clazz":"PaimonDLFExternalCatalog","id":10033,"name":"g3_paimon_dlf","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"dlf","warehouse":"oss://b/w","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10034":{"clazz":"LakeSoulExternalCatalog","id":10034,"name":"g3_lakesoul","logType":"LAKESOUL","catalogProperty":{"resource":"","properties":{"type":"lakesoul","lakesoul.pg.url":"jdbc:postgresql://pg:5432/lakesoul","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""}}} \ No newline at end of file diff --git a/fe/fe-core/src/test/resources/upgrade/413/catalogMgr.json b/fe/fe-core/src/test/resources/upgrade/413/catalogMgr.json new file mode 100644 index 00000000000000..545125a85247aa --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/413/catalogMgr.json @@ -0,0 +1 @@ +{"idToCatalog":{"0":{"clazz":"InternalCatalog"},"10001":{"clazz":"HMSExternalCatalog","id":10001,"name":"g1_hms","logType":"HMS","catalogProperty":{"properties":{"type":"hms","hive.metastore.uris":"thrift://hms-host:9083","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false","ipc.client.fallback-to-simple-auth-allowed":"true"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":[[{"first":"db1","second":"tbl1"},"enable"],[{"first":"db2","second":"tbl2"},"disable"]],"comment":"my hive"},"10002":{"clazz":"IcebergHMSExternalCatalog","id":10002,"name":"g1_ice_hms","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"hms","hive.metastore.uris":"thrift://hms-host:9083","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10003":{"clazz":"IcebergRestExternalCatalog","id":10003,"name":"g1_ice_rest","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"rest","uri":"http://rest:8181","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10004":{"clazz":"IcebergGlueExternalCatalog","id":10004,"name":"g1_ice_glue","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"glue","glue.region":"us-east-1","warehouse":"s3://b/w","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10005":{"clazz":"IcebergDLFExternalCatalog","id":10005,"name":"g1_ice_dlf","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"dlf","dlf.region":"cn-beijing","hive.metastore.type":"dlf","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10006":{"clazz":"IcebergHadoopExternalCatalog","id":10006,"name":"g1_ice_hadoop","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"hadoop","warehouse":"hdfs://nn:8020/wh","fs.defaultFS":"hdfs://nn:8020","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10007":{"clazz":"IcebergJdbcExternalCatalog","id":10007,"name":"g1_ice_jdbc","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"jdbc","uri":"jdbc:postgresql://pg:5432/ice","warehouse":"s3://b/w","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10008":{"clazz":"IcebergS3TablesExternalCatalog","id":10008,"name":"g1_ice_s3tables","logType":"ICEBERG","catalogProperty":{"properties":{"type":"iceberg","iceberg.catalog.type":"s3tables","warehouse":"arn:aws:s3tables:us-east-1:1:bucket/b","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10009":{"clazz":"PaimonExternalCatalog","id":10009,"name":"g1_paimon","logType":"PAIMON","catalogProperty":{"properties":{"type":"paimon","warehouse":"file:/tmp/paimon_wh","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":"paimon fs"},"10010":{"clazz":"EsExternalCatalog","id":10010,"name":"g1_es","logType":"ES","catalogProperty":{"properties":{"enable.mapping.varbinary":"false","use_meta_cache":"true","hosts":"http://es:9200","enable.mapping.timestamp_tz":"false","type":"es"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10011":{"clazz":"JdbcExternalCatalog","id":10011,"name":"g1_jdbc","logType":"JDBC","catalogProperty":{"properties":{"password":"pwd","enable.mapping.varbinary":"false","driver_class":"com.mysql.cj.jdbc.Driver","use_meta_cache":"true","enable.mapping.timestamp_tz":"false","driver_url":"mysql-connector-j-8.3.0.jar","type":"jdbc","user":"root","jdbc_url":"jdbc:mysql://mysql:3306/db?useSSL\u003dfalse\u0026serverTimezone\u003dUTC\u0026yearIsDateType\u003dfalse\u0026tinyInt1isBit\u003dfalse\u0026useUnicode\u003dtrue\u0026rewriteBatchedStatements\u003dtrue\u0026characterEncoding\u003dutf-8"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10012":{"clazz":"MaxComputeExternalCatalog","id":10012,"name":"g1_maxcompute","logType":"MAX_COMPUTE","catalogProperty":{"properties":{"type":"max_compute","mc.default.project":"p","mc.access_key":"ak","mc.secret_key":"sk","mc.endpoint":"http://service.cn.maxcompute.aliyun.com/api","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10013":{"clazz":"TrinoConnectorExternalCatalog","id":10013,"name":"g1_trino","logType":"TRINO_CONNECTOR","catalogProperty":{"properties":{"type":"trino-connector","trino.connector.name":"tpch","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10015":{"clazz":"RemoteDorisExternalCatalog","id":10015,"name":"g1_remote_doris","logType":"REMOTE_DORIS","catalogProperty":{"properties":{"type":"doris","jdbc.jdbc_url":"jdbc:mysql://doris-fe:9030/db","jdbc.user":"root","jdbc.password":"","jdbc.driver_url":"mysql-connector-j-8.3.0.jar","jdbc.driver_class":"com.mysql.cj.jdbc.Driver","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10020":{"clazz":"HMSExternalCatalog","id":10020,"name":"g2_hms_res","logType":"HMS","catalogProperty":{"resource":"hms_res","properties":{"hive.metastore.uris":"thrift://hms-host:9083","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false","ipc.client.fallback-to-simple-auth-allowed":"true"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":"resource backed hms"},"10021":{"clazz":"EsExternalCatalog","id":10021,"name":"g2_es_res","logType":"ES","catalogProperty":{"resource":"es_res","properties":{"enable.mapping.varbinary":"false","use_meta_cache":"true","hosts":"http://es:9200","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10022":{"clazz":"JdbcExternalCatalog","id":10022,"name":"g2_jdbc_res","logType":"JDBC","catalogProperty":{"resource":"jdbc_res","properties":{"password":"pwd","enable.mapping.varbinary":"false","driver_class":"com.mysql.cj.jdbc.Driver","use_meta_cache":"true","enable.mapping.timestamp_tz":"false","driver_url":"mysql-connector-j-8.3.0.jar","user":"root","jdbc_url":"jdbc:mysql://mysql:3306/db?useSSL\u003dfalse\u0026serverTimezone\u003dUTC\u0026yearIsDateType\u003dfalse\u0026tinyInt1isBit\u003dfalse\u0026useUnicode\u003dtrue\u0026rewriteBatchedStatements\u003dtrue\u0026characterEncoding\u003dutf-8"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10023":{"clazz":"TrinoConnectorExternalCatalog","id":10023,"name":"g2_trino_res","logType":"TRINO_CONNECTOR","catalogProperty":{"resource":"trino_res","properties":{"trino.connector.name":"tpch","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10030":{"clazz":"PaimonHMSExternalCatalog","id":10030,"name":"g3_paimon_hms","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"hms","hive.metastore.uris":"thrift://hms-host:9083","warehouse":"hdfs://nn:8020/wh","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10031":{"clazz":"PaimonFileExternalCatalog","id":10031,"name":"g3_paimon_file","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"filesystem","warehouse":"file:/tmp/paimon_wh","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10032":{"clazz":"PaimonRestExternalCatalog","id":10032,"name":"g3_paimon_rest","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"rest","uri":"http://paimon-rest:8080","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10033":{"clazz":"PaimonDLFExternalCatalog","id":10033,"name":"g3_paimon_dlf","logType":"PAIMON","catalogProperty":{"resource":"","properties":{"type":"paimon","paimon.catalog.type":"dlf","warehouse":"oss://b/w","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""},"10034":{"clazz":"LakeSoulExternalCatalog","id":10034,"name":"g3_lakesoul","logType":"LAKESOUL","catalogProperty":{"resource":"","properties":{"type":"lakesoul","lakesoul.pg.url":"jdbc:postgresql://pg:5432/lakesoul","use_meta_cache":"true","enable.mapping.varbinary":"false","enable.mapping.timestamp_tz":"false"}},"initialized":false,"lastUpdateTime":1753000000000,"taap":{},"comment":""}}} \ No newline at end of file diff --git a/fe/fe-core/src/test/resources/upgrade/413/datasource.module.bin b/fe/fe-core/src/test/resources/upgrade/413/datasource.module.bin new file mode 100644 index 00000000000000..a22d7e640cb9ff Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/datasource.module.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/datasource.module.cloud.bin b/fe/fe-core/src/test/resources/upgrade/413/datasource.module.cloud.bin new file mode 100644 index 00000000000000..9d2fe2f5708e0e Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/datasource.module.cloud.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog.index b/fe/fe-core/src/test/resources/upgrade/413/editlog.index new file mode 100644 index 00000000000000..13b6d8194a9127 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/413/editlog.index @@ -0,0 +1,20 @@ +# generated by Gen413Fixtures -- do not edit by hand +op320-create-g1_es.bin +op320-create-g1_hms.bin +op320-create-g1_ice_dlf.bin +op320-create-g1_ice_glue.bin +op320-create-g1_ice_hadoop.bin +op320-create-g1_ice_hms.bin +op320-create-g1_ice_jdbc.bin +op320-create-g1_ice_rest.bin +op320-create-g1_ice_s3tables.bin +op320-create-g1_jdbc.bin +op320-create-g1_maxcompute.bin +op320-create-g1_paimon.bin +op320-create-g1_remote_doris.bin +op320-create-g1_trino.bin +op321-drop.bin +op322-alter-name.bin +op323-alter-props.bin +op324-refresh.bin +op458-alter-comment.bin diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_es.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_es.bin new file mode 100644 index 00000000000000..996a00e9b380fc Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_es.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_hms.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_hms.bin new file mode 100644 index 00000000000000..b9034f0e8c7b55 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_hms.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_dlf.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_dlf.bin new file mode 100644 index 00000000000000..74dcdad5bc9134 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_dlf.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_glue.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_glue.bin new file mode 100644 index 00000000000000..35f137e784a3ac Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_glue.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_hadoop.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_hadoop.bin new file mode 100644 index 00000000000000..911d13075de6a0 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_hadoop.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_hms.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_hms.bin new file mode 100644 index 00000000000000..dac56bf8db60cc Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_hms.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_jdbc.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_jdbc.bin new file mode 100644 index 00000000000000..3b67d5b6f2858b Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_jdbc.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_rest.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_rest.bin new file mode 100644 index 00000000000000..4d7f29ae0e6d74 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_rest.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_s3tables.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_s3tables.bin new file mode 100644 index 00000000000000..543c88315fb341 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_ice_s3tables.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_jdbc.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_jdbc.bin new file mode 100644 index 00000000000000..0dc1e36885410a Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_jdbc.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_maxcompute.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_maxcompute.bin new file mode 100644 index 00000000000000..02b7d6c7f96d4a Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_maxcompute.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_paimon.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_paimon.bin new file mode 100644 index 00000000000000..a5950fc5e74e90 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_paimon.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_remote_doris.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_remote_doris.bin new file mode 100644 index 00000000000000..8dbf38bf149f51 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_remote_doris.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_trino.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_trino.bin new file mode 100644 index 00000000000000..40e650918ea766 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op320-create-g1_trino.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op321-drop.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op321-drop.bin new file mode 100644 index 00000000000000..79f3d6aa8a232c Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op321-drop.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op322-alter-name.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op322-alter-name.bin new file mode 100644 index 00000000000000..581ec8cc21a5d3 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op322-alter-name.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op323-alter-props.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op323-alter-props.bin new file mode 100644 index 00000000000000..952242a5c34c3d Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op323-alter-props.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op324-refresh.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op324-refresh.bin new file mode 100644 index 00000000000000..e65c76b0014724 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op324-refresh.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/editlog/op458-alter-comment.bin b/fe/fe-core/src/test/resources/upgrade/413/editlog/op458-alter-comment.bin new file mode 100644 index 00000000000000..deda18b5738126 Binary files /dev/null and b/fe/fe-core/src/test/resources/upgrade/413/editlog/op458-alter-comment.bin differ diff --git a/fe/fe-core/src/test/resources/upgrade/413/generator/Gen413Fixtures.java b/fe/fe-core/src/test/resources/upgrade/413/generator/Gen413Fixtures.java new file mode 100644 index 00000000000000..beddf29111c891 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/413/generator/Gen413Fixtures.java @@ -0,0 +1,406 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tools; + +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.io.Text; +import org.apache.doris.datasource.CatalogFactory; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogLog; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.es.EsExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; +import org.apache.doris.datasource.lakesoul.LakeSoulExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonDLFExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonHMSExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonRestExternalCatalog; +import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalog; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.persist.gson.RuntimeTypeAdapterFactory; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Generates golden metadata fixtures EXACTLY as Doris 4.1.3 writes them. + * + *

This class is NOT part of Doris. It lives only in a throwaway git worktree checked out at the + * 4.1.3 tag, so that the fixtures it emits are produced by real 4.1.3 bytecode rather than by + * someone's idea of what 4.1.3 writes. See plan-doc/41-upgrade-compat-test-plan.md.

+ * + *

Usage: {@code java -cp org.apache.doris.tools.Gen413Fixtures [cloud]}

+ */ +public class Gen413Fixtures { + + /** Fixed so the output is byte-deterministic across runs. */ + private static final long FIXED_UPDATE_TIME = 1753000000000L; + + private final List> catalogs = new ArrayList<>(); + private final List failures = new ArrayList<>(); + + public static void main(String[] args) throws Exception { + if (args.length < 1) { + throw new IllegalArgumentException("usage: Gen413Fixtures [cloud]"); + } + File outDir = new File(args[0]); + boolean cloud = args.length > 1 && "cloud".equals(args[1]); + // MUST happen before GsonUtils / EnvFactory class-init: both latch the mode in static initializers. + if (cloud) { + Config.deploy_mode = "cloud"; + } + FeConstants.runningUnitTest = true; + new Gen413Fixtures().run(outDir, cloud); + } + + private void run(File outDir, boolean cloud) throws Exception { + if (!outDir.exists() && !outDir.mkdirs()) { + throw new IllegalStateException("cannot create " + outDir); + } + buildG1(); + buildG2(); + buildG3(); + + CatalogMgr mgr = newCatalogMgrWith(catalogs); + String json = GsonUtils.GSON.toJson(mgr); + + String suffix = cloud ? ".cloud" : ""; + writeFramed(new File(outDir, "datasource.module" + suffix + ".bin"), json); + Files.write(Paths.get(new File(outDir, "catalogMgr" + suffix + ".json").toURI()), + json.getBytes(StandardCharsets.UTF_8)); + + if (!cloud) { + dumpLabels(outDir); + buildEditLogFixtures(new File(outDir, "editlog")); + } + + System.out.println("=== catalogs written: " + catalogs.size() + " (cloud=" + cloud + ")"); + for (CatalogIf c : catalogs) { + System.out.println(" " + c.getId() + " " + c.getClass().getSimpleName() + " " + c.getName()); + } + System.out.println("=== module sha256: " + sha256(json.getBytes(StandardCharsets.UTF_8))); + if (!failures.isEmpty()) { + System.out.println("=== FAILURES (" + failures.size() + "):"); + failures.forEach(f -> System.out.println(" " + f)); + throw new IllegalStateException(failures.size() + " fixture(s) could not be built"); + } + } + + // --------------------------------------------------------------------------------------------------- + // G1 -- built through 4.1.3's own CatalogFactory replay path, props DO carry "type". + // This is what a catalog created ON 4.1.3 looks like. + // --------------------------------------------------------------------------------------------------- + private void buildG1() { + g1(10001, "g1_hms", "my hive", props("type", "hms", + "hive.metastore.uris", "thrift://hms-host:9083"), true); + g1(10002, "g1_ice_hms", "", props("type", "iceberg", + "iceberg.catalog.type", "hms", "hive.metastore.uris", "thrift://hms-host:9083"), false); + g1(10003, "g1_ice_rest", "", props("type", "iceberg", + "iceberg.catalog.type", "rest", "uri", "http://rest:8181"), false); + g1(10004, "g1_ice_glue", "", props("type", "iceberg", + "iceberg.catalog.type", "glue", "glue.region", "us-east-1", "warehouse", "s3://b/w"), false); + g1(10005, "g1_ice_dlf", "", props("type", "iceberg", + "iceberg.catalog.type", "dlf", "dlf.region", "cn-beijing"), false); + g1(10006, "g1_ice_hadoop", "", props("type", "iceberg", + "iceberg.catalog.type", "hadoop", "warehouse", "hdfs://nn:8020/wh"), false); + g1(10007, "g1_ice_jdbc", "", props("type", "iceberg", + "iceberg.catalog.type", "jdbc", "uri", "jdbc:postgresql://pg:5432/ice", "warehouse", "s3://b/w"), false); + g1(10008, "g1_ice_s3tables", "", props("type", "iceberg", + "iceberg.catalog.type", "s3tables", "warehouse", "arn:aws:s3tables:us-east-1:1:bucket/b"), false); + g1(10009, "g1_paimon", "paimon fs", props("type", "paimon", + "warehouse", "file:/tmp/paimon_wh"), false); + g1(10010, "g1_es", "", props("type", "es", "hosts", "http://es:9200"), false); + // jdbc_url deliberately carries '=' and '&': they are HTML-escaped on the wire (= / &). + g1(10011, "g1_jdbc", "", props("type", "jdbc", + "user", "root", "password", "pwd", + "jdbc_url", "jdbc:mysql://mysql:3306/db?useSSL=false&serverTimezone=UTC", + "driver_url", "mysql-connector-j-8.3.0.jar", + "driver_class", "com.mysql.cj.jdbc.Driver"), false); + g1(10012, "g1_maxcompute", "", props("type", "max_compute", + "mc.default.project", "p", "mc.access_key", "ak", "mc.secret_key", "sk", + "mc.endpoint", "http://service.cn.maxcompute.aliyun.com/api"), false); + g1(10013, "g1_trino", "", props("type", "trino-connector", + "trino.connector.name", "tpch"), false); + // Control group: the engine implements this type itself, so after the SPI cutover it must KEEP + // its own class and must NOT be remapped onto PluginDrivenExternalCatalog. + // ("test" is deliberately absent: TestExternalCatalog needs a UT-only provider class and can + // never appear in a production 4.1.3 image.) + g1(10015, "g1_remote_doris", "", props("type", "doris", + "jdbc.jdbc_url", "jdbc:mysql://doris-fe:9030/db", + "jdbc.user", "root", "jdbc.password", "", + "jdbc.driver_url", "mysql-connector-j-8.3.0.jar", + "jdbc.driver_class", "com.mysql.cj.jdbc.Driver"), false); + } + + private void g1(long id, String name, String comment, Map props, boolean withTaap) { + try { + CatalogLog log = new CatalogLog(); + log.setCatalogId(id); + log.setCatalogName(name); + log.setComment(comment); + log.setProps(props); + CatalogIf c = CatalogFactory.createFromLog(log); + decorate(c, withTaap); + catalogs.add(c); + } catch (Throwable t) { + failures.add("G1 " + name + ": " + t); + } + } + + // --------------------------------------------------------------------------------------------------- + // G2 -- resource-backed shape: props carry NO "type" key, catalogProperty.resource is set. + // 4.1.3 did not need "type" because the concrete class carried it; after the SPI cutover the type + // can only be recovered from logType. This family is the ONLY one that can fail when HEAD's + // PluginDrivenExternalCatalog.gsonPostProcess backfill is removed. + // Built by calling the concrete ctor + setDefaultPropsIfMissing(true), which is literally what + // 4.1.3's CatalogFactory switch does after picking the class. + // --------------------------------------------------------------------------------------------------- + private void buildG2() { + g2(10020, "g2_hms_res", () -> new HMSExternalCatalog(10020, "g2_hms_res", "hms_res", + props("hive.metastore.uris", "thrift://hms-host:9083"), "resource backed hms")); + g2(10021, "g2_es_res", () -> new EsExternalCatalog(10021, "g2_es_res", "es_res", + props("hosts", "http://es:9200"), "")); + g2(10022, "g2_jdbc_res", () -> new JdbcExternalCatalog(10022, "g2_jdbc_res", "jdbc_res", + props("user", "root", "password", "pwd", + "jdbc_url", "jdbc:mysql://mysql:3306/db?useSSL=false&serverTimezone=UTC", + "driver_url", "mysql-connector-j-8.3.0.jar", + "driver_class", "com.mysql.cj.jdbc.Driver"), "")); + // The load-bearing one: TRINO_CONNECTOR is the only logType whose name().toLowerCase() + // ("trino_connector") differs from the catalog type string the connector answers to + // ("trino-connector"). + g2(10023, "g2_trino_res", () -> new TrinoConnectorExternalCatalog(10023, "g2_trino_res", "trino_res", + props("trino.connector.name", "tpch"), "")); + } + + // --------------------------------------------------------------------------------------------------- + // G3 -- labels that exist in 4.1.3's GsonUtils and can appear in a 4.1 image, but that 4.1.3's own + // factories can no longer produce (the paimon factory always returns the base PaimonExternalCatalog; + // lakesoul creation was already rejected). They are carry-forward from <= 4.0 metadata. + // Still real 4.1.3 bytecode -- only the construction route differs. Recorded in PROVENANCE.txt. + // --------------------------------------------------------------------------------------------------- + private void buildG3() { + g2(10030, "g3_paimon_hms", () -> new PaimonHMSExternalCatalog(10030, "g3_paimon_hms", "", + props("type", "paimon", "paimon.catalog.type", "hms", + "hive.metastore.uris", "thrift://hms-host:9083", "warehouse", "hdfs://nn:8020/wh"), "")); + g2(10031, "g3_paimon_file", () -> new PaimonFileExternalCatalog(10031, "g3_paimon_file", "", + props("type", "paimon", "paimon.catalog.type", "filesystem", + "warehouse", "file:/tmp/paimon_wh"), "")); + g2(10032, "g3_paimon_rest", () -> new PaimonRestExternalCatalog(10032, "g3_paimon_rest", "", + props("type", "paimon", "paimon.catalog.type", "rest", "uri", "http://paimon-rest:8080"), "")); + g2(10033, "g3_paimon_dlf", () -> new PaimonDLFExternalCatalog(10033, "g3_paimon_dlf", "", + props("type", "paimon", "paimon.catalog.type", "dlf", "warehouse", "oss://b/w"), "")); + g2(10034, "g3_lakesoul", () -> new LakeSoulExternalCatalog(10034, "g3_lakesoul", "", + props("type", "lakesoul", "lakesoul.pg.url", "jdbc:postgresql://pg:5432/lakesoul"), "")); + } + + private interface CatalogSupplier { + ExternalCatalog get() throws Exception; + } + + private void g2(long id, String name, CatalogSupplier supplier) { + try { + ExternalCatalog c = supplier.get(); + // exactly what 4.1.3's CatalogFactory does right after picking the concrete class + c.setDefaultPropsIfMissing(true); + decorate(c, false); + catalogs.add(c); + } catch (Throwable t) { + failures.add("G2/G3 " + name + " (id=" + id + "): " + t); + } + } + + private void decorate(CatalogIf c, boolean withTaap) { + if (!(c instanceof ExternalCatalog)) { + return; + } + ExternalCatalog ec = (ExternalCatalog) c; + ec.setLastUpdateTime(FIXED_UPDATE_TIME); + if (withTaap) { + // Exercises the polymorphic taap encoding: empty map serializes as {}, non-empty as + // an array of [key, value] 2-tuples (enableComplexMapKeySerialization). + ec.setAutoAnalyzePolicy("db1", "tbl1", "enable"); + ec.setAutoAnalyzePolicy("db2", "tbl2", "disable"); + } + } + + // --------------------------------------------------------------------------------------------------- + // Edit-log fixtures: JournalEntity wire form is writeShort(opCode) + payload.write(out), + // and CatalogLog.write is Text.writeString(out, GSON.toJson(this)). Nothing about the class is + // on the wire, so a 4.1.3 entry and a HEAD entry are byte-identical -- only replay behaviour differs. + // --------------------------------------------------------------------------------------------------- + private void buildEditLogFixtures(File dir) throws Exception { + if (!dir.exists() && !dir.mkdirs()) { + throw new IllegalStateException("cannot create " + dir); + } + // OP_CREATE_CATALOG built from the real catalogs via CatalogIf.constructEditLog(), + // i.e. exactly what 4.1.3's CatalogMgr.createCatalog would have logged. + // + // G1 ONLY, on purpose. constructEditLog() hardcodes setResource("") and journals nothing but the + // property map, so a resource-backed catalog (G2) would produce an entry carrying neither a resource + // nor a "type" -- a shape 4.1.3 can never emit, because 4.1.3 has no CREATE CATALOG ... WITH RESOURCE + // and therefore never journals such a catalog in the first place. G2 reaches an upgraded FE through + // the image, never through the journal. G3 is likewise unjournalable (the paimon factory always + // returns the base class, lakesoul creation is rejected outright) and its props are indistinguishable + // from G1's on the wire anyway, since the class name is not part of a journal entry. + for (CatalogIf c : catalogs) { + if (!(c instanceof ExternalCatalog) || !c.getName().startsWith("g1_")) { + continue; + } + writeOp(dir, 320, "create-" + c.getName(), c.constructEditLog()); + } + CatalogLog drop = new CatalogLog(); + drop.setCatalogId(10001); + writeOp(dir, 321, "drop", drop); + + CatalogLog rename = new CatalogLog(); + rename.setCatalogId(10001); + rename.setNewCatalogName("g1_hms_renamed"); + writeOp(dir, 322, "alter-name", rename); + + CatalogLog alterProps = new CatalogLog(); + alterProps.setCatalogId(10001); + alterProps.setNewProps(props("hive.metastore.uris", "thrift://other-hms:9083", + "metadata_refresh_interval_sec", "30")); + writeOp(dir, 323, "alter-props", alterProps); + + CatalogLog refresh = new CatalogLog(); + refresh.setCatalogId(10001); + refresh.setInvalidCache(true); + writeOp(dir, 324, "refresh", refresh); + + CatalogLog comment = new CatalogLog(); + comment.setCatalogId(10001); + comment.setComment("new comment"); + writeOp(dir, 458, "alter-comment", comment); + + // Flat index: resource directories cannot be enumerated from a jar, so the tests read this + // instead of listing the directory. Regenerating the fixtures regenerates the index, which is + // why it lives here rather than being maintained by hand. + String[] names = dir.list(); + if (names == null) { + throw new IllegalStateException("cannot list " + dir); + } + java.util.Arrays.sort(names); + StringBuilder sb = new StringBuilder("# generated by Gen413Fixtures -- do not edit by hand\n"); + for (String n : names) { + sb.append(n).append('\n'); + } + Files.write(Paths.get(new File(dir.getParentFile(), "editlog.index").toURI()), + sb.toString().getBytes(StandardCharsets.UTF_8)); + } + + private void writeOp(File dir, int opCode, String name, CatalogLog log) throws Exception { + File f = new File(dir, String.format("op%d-%s.bin", opCode, name)); + try (DataOutputStream out = new DataOutputStream(new FileOutputStream(f))) { + out.writeShort(opCode); + log.write(out); + } + } + + // --------------------------------------------------------------------------------------------------- + // Label inventory, read reflectively out of 4.1.3's own registry. Machine-generated on purpose: + // a hand-written or regex-scraped list silently drops entries (e.g. IcebergS3TablesExternalCatalog, + // whose class name contains a digit). + // --------------------------------------------------------------------------------------------------- + private void dumpLabels(File outDir) throws Exception { + dumpOneRegistry(outDir, "labels.ds.txt", "dsTypeAdapterFactory"); + dumpOneRegistry(outDir, "labels.db.txt", "dbTypeAdapterFactory"); + dumpOneRegistry(outDir, "labels.tbl.txt", "tblTypeAdapterFactory"); + } + + private void dumpOneRegistry(File outDir, String fileName, String fieldName) throws Exception { + Field f = GsonUtils.class.getDeclaredField(fieldName); + f.setAccessible(true); + RuntimeTypeAdapterFactory factory = (RuntimeTypeAdapterFactory) f.get(null); + Field l = RuntimeTypeAdapterFactory.class.getDeclaredField("labelToSubtype"); + l.setAccessible(true); + @SuppressWarnings("unchecked") + Map> labelToSubtype = (Map>) l.get(factory); + StringBuilder sb = new StringBuilder(); + sb.append("# generated from 4.1.3 by Gen413Fixtures -- do not edit by hand\n"); + sb.append("#