Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[AMORO-2862] fix bug: A null pointer error occurs when using S3 as Storage when using the kyuubi terminal. #2863

Merged
merged 8 commits into from
May 30, 2024
Merged

[AMORO-2862] fix bug: A null pointer error occurs when using S3 as Storage when using the kyuubi terminal. #2863

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
32c77d5
[AMORO-2862] Fix bug #2862 A null pointer error occurs when using S3 …
May 24, 2024
b71fc92
[AMORO-2862] Fix bug #2862 A null pointer error occurs when using S3 …
May 24, 2024
62af715
[AMORO-2862] Fix bug #2862 A null pointer error occurs when using S3 …
May 24, 2024
6a26389
Merge branch 'master' into AMORO-2862
zhoujinsong May 27, 2024
b9e9e26
[AMORO-2862] Fix bug #2862 A null pointer error occurs when using S3 …
May 28, 2024
220b800
[AMORO-2862] Fix bug #2862 A null pointer error occurs when using S3 …
May 28, 2024
3d968e1
Merge branch 'master' into AMORO-2862
baiyangtx May 28, 2024
9fa709b
[AMORO-2862] Fix bug #2862 A null pointer error occurs when using S3 …
May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/terminal/TerminalManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,10 @@ private TableMetaStore getCatalogTableMetaStore(CatalogMeta catalogMeta) {
catalogMeta
.getStorageConfigs()
.get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE));
} else if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3.equalsIgnoreCase(
MixedCatalogUtil.getCompatibleStorageType(storageConfigs))) {
builder.withStorageType(
catalogMeta.getStorageConfigs().get(CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE));
}
}
String authType = catalogMeta.getAuthConfigs().get(CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE);
Expand Down
8 changes: 6 additions & 2 deletions ...r/src/main/java/org/apache/amoro/server/terminal/kyuubi/KyuubiTerminalSessionFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.amoro.api.config.ConfigOption;
import org.apache.amoro.api.config.ConfigOptions;
import org.apache.amoro.api.config.Configurations;
import org.apache.amoro.properties.CatalogMetaProperties;
import org.apache.amoro.server.terminal.SparkContextUtil;
import org.apache.amoro.server.terminal.TerminalSession;
import org.apache.amoro.server.terminal.TerminalSessionFactory;
Expand Down Expand Up @@ -124,8 +125,11 @@ public TerminalSession create(TableMetaStore metaStore, Configurations configura
Properties properties = new Properties();

if (!metaStore.isKerberosAuthMethod()) {
properties.put(JdbcConnectionParams.AUTH_USER, metaStore.getHadoopUsername());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about checking if the value is null before putting it into properties and configuration?
I think it will be simpler if we fix it this way.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for your suggestion, I have made the changes.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can use one condition to fix this, WDYT

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can use one condition to fix this, WDYT

Okay, I've merged it into one.thinks.

sessionConf.put(JdbcConnectionParams.AUTH_USER, metaStore.getHadoopUsername());
if (!CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_S3.equalsIgnoreCase(
metaStore.getStorageType())) {
properties.put(JdbcConnectionParams.AUTH_USER, metaStore.getHadoopUsername());
sessionConf.put(JdbcConnectionParams.AUTH_USER, metaStore.getHadoopUsername());
}
}

Connection connection = metaStore.doAs(() -> driver.connect(kyuubiJdbcUrl, properties));
Expand Down
16 changes: 15 additions & 1 deletion amoro-core/src/main/java/org/apache/amoro/table/TableMetaStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ public class TableMetaStore implements Serializable {
public static final String SIMPLE_USER_NAME = "simple.user.name";
public static final String AUTH_METHOD_SIMPLE = "SIMPLE";
public static final String AUTH_METHOD_KERBEROS = "KERBEROS";

private static final String KRB_CONF_FILE_NAME = "krb5.conf";
private static final String KEY_TAB_FILE_NAME = "krb.keytab";
private static final String META_STORE_SITE_FILE_NAME = "hive-site.xml";
Expand Down Expand Up @@ -117,6 +116,7 @@ public class TableMetaStore implements Serializable {
private final byte[] hdfsSite;
private final byte[] coreSite;
private final String authMethod;
private final String storageType;
private final String hadoopUsername;
private final byte[] krbKeyTab;
private final byte[] krbConf;
Expand All @@ -132,6 +132,7 @@ public static Builder builder() {

private TableMetaStore(
byte[] metaStoreSite,
String storageType,
byte[] hdfsSite,
byte[] coreSite,
String authMethod,
Expand All @@ -147,6 +148,7 @@ private TableMetaStore(
"Error auth method:%s",
authMethod);
this.metaStoreSite = metaStoreSite;
this.storageType = storageType;
this.hdfsSite = hdfsSite;
this.coreSite = coreSite;
this.authMethod = authMethod;
Expand All @@ -161,6 +163,10 @@ public byte[] getMetaStoreSite() {
return metaStoreSite;
}

public String getStorageType() {
return storageType;
}

public byte[] getHdfsSite() {
return hdfsSite;
}
Expand Down Expand Up @@ -549,6 +555,7 @@ public static class Builder {
private byte[] hdfsSite;
private byte[] coreSite;
private String authMethod;
private String storageType;
private String hadoopUsername;
private byte[] krbKeyTab;
private byte[] krbConf;
Expand All @@ -562,6 +569,11 @@ public Builder withMetaStoreSitePath(String metaStoreSitePath) {
return this;
}

public Builder withStorageType(String storageType) {
this.storageType = storageType;
return this;
}

public Builder withMetaStoreSite(byte[] metaStoreSiteBytes) {
this.metaStoreSite = metaStoreSiteBytes;
return this;
Expand Down Expand Up @@ -748,6 +760,7 @@ public TableMetaStore build() {
krbPrincipal);
return new TableMetaStore(
metaStoreSite,
storageType,
hdfsSite,
coreSite,
authMethod,
Expand All @@ -764,6 +777,7 @@ public TableMetaStore buildForTest() {
TableMetaStore tableMetaStore =
new TableMetaStore(
metaStoreSite,
storageType,
hdfsSite,
coreSite,
authMethod,
Expand Down
Loading