Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@
import org.apache.hudi.aws.testutils.GlueTestUtil;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.sync.common.HoodieSyncTool;
import org.apache.hudi.sync.common.util.SyncUtilHelpers;

import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -38,6 +40,7 @@
import software.amazon.awssdk.services.sts.model.GetCallerIdentityResponse;

import java.io.IOException;
import java.util.Properties;

import static org.apache.hudi.aws.testutils.GlueTestUtil.getHadoopConf;
import static org.apache.hudi.aws.testutils.GlueTestUtil.glueSyncProps;
Expand Down Expand Up @@ -105,4 +108,32 @@ void validateInitThroughSyncTool() throws Exception {
syncTool.close();
}
}

/**
* Verifies the 3-arg {@code (Properties, Configuration, Option)} reflection path used by Flink's
* {@code HiveSyncContext#hiveSyncTool()} can fully instantiate {@link AwsGlueCatalogSyncTool}.
*/
@Test
void validateInitThroughHiveSyncContextReflectionSignature() throws Exception {
try (MockedStatic<GlueAsyncClient> mockedStatic = mockStatic(GlueAsyncClient.class);
MockedStatic<StsClient> mockedStsStatic = mockStatic(StsClient.class)) {
GlueAsyncClientBuilder builder = mock(GlueAsyncClientBuilder.class);
mockedStatic.when(GlueAsyncClient::builder).thenReturn(builder);
when(builder.credentialsProvider(any())).thenReturn(builder);
GlueAsyncClient mockClient = mock(GlueAsyncClient.class);
when(builder.build()).thenReturn(mockClient);
StsClient mockSts = mock(StsClient.class);
mockedStsStatic.when(StsClient::create).thenReturn(mockSts);
when(mockSts.getCallerIdentity(GetCallerIdentityRequest.builder().build()))
.thenReturn(GetCallerIdentityResponse.builder().account("").build());

Object syncTool = ReflectionUtils.loadClass(
AwsGlueCatalogSyncTool.class.getName(),
new Class<?>[] {Properties.class, Configuration.class, Option.class},
glueSyncProps, getHadoopConf(), Option.empty());
assertTrue(syncTool instanceof AwsGlueCatalogSyncTool,
"Reflection through the HiveSyncContext signature must yield an AwsGlueCatalogSyncTool instance");
((AwsGlueCatalogSyncTool) syncTool).close();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
package org.apache.hudi.sink.utils;

import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
Expand Down Expand Up @@ -78,8 +80,8 @@ public HiveSyncTool hiveSyncTool() {
HiveSyncMode syncMode = HiveSyncMode.of(props.getProperty(HIVE_SYNC_MODE.key()));
if (syncMode == HiveSyncMode.GLUE) {
return ((HiveSyncTool) ReflectionUtils.loadClass(AWS_GLUE_CATALOG_SYNC_TOOL_CLASS,
new Class<?>[] {Properties.class, org.apache.hadoop.conf.Configuration.class},
props, hiveConf));
new Class<?>[] {Properties.class, org.apache.hadoop.conf.Configuration.class, Option.class},
props, hiveConf, Option.<HoodieTableMetaClient>empty()));
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do you think we can write a simple UT of HiveSyncContext to validate the instantiation of the AWS_GLUE_CATALOG_SYNC_TOOL_CLASS ?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 nit: the <HoodieTableMetaClient> type witness isn't needed here — ReflectionUtils.loadClass takes Object... args, so Option.empty() works fine and would also let you drop the HoodieTableMetaClient import that was added solely for this.

- AI-generated; verify before applying. React 👍/👎 to flag quality.

}
return new HiveSyncTool(props, hiveConf);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.hudi.sink.utils;

import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.hive.HiveSyncConfig;

Expand Down Expand Up @@ -66,4 +68,18 @@ void testOptionWithoutShortcutKey() {
Properties props3 = HiveSyncContext.buildSyncConfig(configuration3);
assertTrue(Boolean.parseBoolean(props3.getProperty(HiveSyncConfig.HIVE_CREATE_MANAGED_TABLE.key(), "false")));
}

/**
* Pins the constructor signature {@link HiveSyncContext#hiveSyncTool()} relies on via reflection.
* End-to-end instantiation is covered in {@code hudi-aws}'s {@code TestAwsGlueSyncTool}.
*/
@Test
void testAwsGlueSyncToolReflectionConstructorExists() {
assertTrue(
ReflectionUtils.hasConstructor(
HiveSyncContext.AWS_GLUE_CATALOG_SYNC_TOOL_CLASS,
new Class<?>[] {Properties.class, org.apache.hadoop.conf.Configuration.class, Option.class}),
"AwsGlueCatalogSyncTool must expose the constructor used by HiveSyncContext#hiveSyncTool() "
+ "via reflection; otherwise Flink GLUE sync fails with NoSuchMethodException.");
}
}
Loading