From c72571503cfbc194dcb51da1aa3226b46df3f86e Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:43:23 +0000 Subject: [PATCH 1/9] add user-agent parameter --- README.md | 3 +++ .../org/embulk/output/DatabricksOutputPlugin.java | 12 ++++++++++++ .../output/databricks/DatabricksAPIClient.java | 10 ++++++++++ 3 files changed, 25 insertions(+) diff --git a/README.md b/README.md index 16163f7..0d33284 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Databricks output plugin for Embulk loads records to Databricks Delta Table. - **driver_path**: path to the jar file of the JDBC driver. If not set, [the bundled JDBC driver](https://docs.databricks.com/en/integrations/jdbc/index.html) will be used. (string, optional) - **options**: extra JDBC properties (hash, default: {}) +- **user_agent**: set user agent property to JDBC and SDK connection. If 'UserAgentEntry' property is specified in the **options**, it will be overwritten by this value. (hash, optional) + - **product_name**: product name of user agent (string) + - **product_version**: product version of user agent (string) - **server_hostname**: The Databricks compute resource’s Server Hostname value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **http_path**: The Databricks compute resource’s HTTP Path value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **personal_access_token**: The Databaricks personal_access_token, see [Authentication settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/authentication.html#authentication-pat). (string, required) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index 49132ad..7b611a3 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -52,6 +52,10 @@ public interface DatabricksPluginTask extends PluginTask { @Config("delete_stage_on_error") @ConfigDefault("false") public boolean getDeleteStageOnError(); + + @Config("user_agent") + @ConfigDefault("null") + public Optional> getUserAgent(); } @Override @@ -92,6 +96,14 @@ protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMet props.put("ConnCatalog", t.getCatalogName()); props.put("ConnSchema", t.getSchemaName()); props.putAll(t.getOptions()); + // overwrite UserAgentEntry property if the same property is set in options + if (t.getUserAgent().isPresent()) { + String product_name = t.getUserAgent().get().get("product_name"); + String product_version = t.getUserAgent().get().get("product_version"); + + props.put("UserAgentEntry", product_name + "/" + product_version); + } + logConnectionProperties(url, props); return new DatabricksOutputConnector( url, props, t.getTransactionIsolation(), t.getCatalogName(), t.getSchemaName()); diff --git a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java index ef23dbb..bd89ef3 100644 --- a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java +++ b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java @@ -2,6 +2,7 @@ import com.databricks.sdk.WorkspaceClient; import com.databricks.sdk.core.DatabricksConfig; +import com.databricks.sdk.core.UserAgent; import com.databricks.sdk.service.catalog.VolumeType; import java.io.InputStream; import java.text.SimpleDateFormat; @@ -11,9 +12,18 @@ public class DatabricksAPIClient { public static DatabricksAPIClient create(DatabricksOutputPlugin.DatabricksPluginTask task) { + SetUserAgent(task); + return new DatabricksAPIClient(createDatabricksConfig(task)); } + private static void SetUserAgent(DatabricksOutputPlugin.DatabricksPluginTask task) { + String name = task.getUserAgent().get().get("product_name"); + String version = task.getUserAgent().get().get("product_version"); + + UserAgent.withProduct(name, version); + } + private final WorkspaceClient workspaceClient; public DatabricksAPIClient(DatabricksConfig config) { From 0a4f81eb8078f2326b60fcf236afaa3c05d4dc74 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 01:20:11 +0000 Subject: [PATCH 2/9] fix method name to start with lowercase --- .../org/embulk/output/databricks/DatabricksAPIClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java index bd89ef3..b07858b 100644 --- a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java +++ b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java @@ -12,12 +12,12 @@ public class DatabricksAPIClient { public static DatabricksAPIClient create(DatabricksOutputPlugin.DatabricksPluginTask task) { - SetUserAgent(task); + setUserAgent(task); return new DatabricksAPIClient(createDatabricksConfig(task)); } - private static void SetUserAgent(DatabricksOutputPlugin.DatabricksPluginTask task) { + private static void setUserAgent(DatabricksOutputPlugin.DatabricksPluginTask task) { String name = task.getUserAgent().get().get("product_name"); String version = task.getUserAgent().get().get("product_version"); From e92dae6d64fc995d92a29f0bbd0014f3fba26e29 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 03:03:30 +0000 Subject: [PATCH 3/9] add default value to user-agent parameter --- README.md | 4 ++-- .../embulk/output/DatabricksOutputPlugin.java | 19 +++++++++++++++---- .../databricks/DatabricksAPIClient.java | 4 ++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0d33284..a13f641 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ Databricks output plugin for Embulk loads records to Databricks Delta Table. - **driver_path**: path to the jar file of the JDBC driver. If not set, [the bundled JDBC driver](https://docs.databricks.com/en/integrations/jdbc/index.html) will be used. (string, optional) - **options**: extra JDBC properties (hash, default: {}) - **user_agent**: set user agent property to JDBC and SDK connection. If 'UserAgentEntry' property is specified in the **options**, it will be overwritten by this value. (hash, optional) - - **product_name**: product name of user agent (string) - - **product_version**: product version of user agent (string) + - **product_name**: product name of user agent (string, default: unknown) + - **product_version**: product version of user agent (string, default: 0.0.0) - **server_hostname**: The Databricks compute resource’s Server Hostname value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **http_path**: The Databricks compute resource’s HTTP Path value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **personal_access_token**: The Databaricks personal_access_token, see [Authentication settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/authentication.html#authentication-pat). (string, required) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index 7b611a3..a354816 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -15,6 +15,7 @@ import org.embulk.spi.Schema; import org.embulk.util.config.Config; import org.embulk.util.config.ConfigDefault; +import org.embulk.util.config.Task; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +55,18 @@ public interface DatabricksPluginTask extends PluginTask { public boolean getDeleteStageOnError(); @Config("user_agent") - @ConfigDefault("null") - public Optional> getUserAgent(); + @ConfigDefault("{}") + public Optional getUserAgent(); + + public interface UserAgentEntry extends Task { + @Config("product_name") + @ConfigDefault("\"unknown\"") + public String getProductName(); + + @Config("product_version") + @ConfigDefault("\"0.0.0\"") + public String getProductVersion(); + } } @Override @@ -98,8 +109,8 @@ protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMet props.putAll(t.getOptions()); // overwrite UserAgentEntry property if the same property is set in options if (t.getUserAgent().isPresent()) { - String product_name = t.getUserAgent().get().get("product_name"); - String product_version = t.getUserAgent().get().get("product_version"); + String product_name = t.getUserAgent().get().getProductName(); + String product_version = t.getUserAgent().get().getProductVersion(); props.put("UserAgentEntry", product_name + "/" + product_version); } diff --git a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java index b07858b..9665e43 100644 --- a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java +++ b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java @@ -18,8 +18,8 @@ public static DatabricksAPIClient create(DatabricksOutputPlugin.DatabricksPlugin } private static void setUserAgent(DatabricksOutputPlugin.DatabricksPluginTask task) { - String name = task.getUserAgent().get().get("product_name"); - String version = task.getUserAgent().get().get("product_version"); + String name = task.getUserAgent().get().getProductName(); + String version = task.getUserAgent().get().getProductVersion(); UserAgent.withProduct(name, version); } From eaba956c431a797e9996ab9be6e962f93756d19b Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 03:05:29 +0000 Subject: [PATCH 4/9] fix readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a13f641..067b194 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ Databricks output plugin for Embulk loads records to Databricks Delta Table. - **driver_path**: path to the jar file of the JDBC driver. If not set, [the bundled JDBC driver](https://docs.databricks.com/en/integrations/jdbc/index.html) will be used. (string, optional) - **options**: extra JDBC properties (hash, default: {}) - **user_agent**: set user agent property to JDBC and SDK connection. If 'UserAgentEntry' property is specified in the **options**, it will be overwritten by this value. (hash, optional) - - **product_name**: product name of user agent (string, default: unknown) - - **product_version**: product version of user agent (string, default: 0.0.0) + - **product_name**: product name of user agent (string, default: "unknown") + - **product_version**: product version of user agent (string, default: "0.0.0") - **server_hostname**: The Databricks compute resource’s Server Hostname value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **http_path**: The Databricks compute resource’s HTTP Path value, see [Compute settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/compute.html). (string, required) - **personal_access_token**: The Databaricks personal_access_token, see [Authentication settings for the Databricks JDBC Driver](https://docs.databricks.com/en/integrations/jdbc/authentication.html#authentication-pat). (string, required) From 0e4ed29571b0345865c38361ed630f50a3e3d09f Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:57:42 +0000 Subject: [PATCH 5/9] remove optional signature and align interface name with getter --- .../java/org/embulk/output/DatabricksOutputPlugin.java | 8 ++++---- .../org/embulk/output/databricks/DatabricksAPIClient.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index a354816..65ec4cb 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -56,7 +56,7 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") - public Optional getUserAgent(); + public UserAgentEntry getUserAgentEntry(); public interface UserAgentEntry extends Task { @Config("product_name") @@ -108,9 +108,9 @@ protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMet props.put("ConnSchema", t.getSchemaName()); props.putAll(t.getOptions()); // overwrite UserAgentEntry property if the same property is set in options - if (t.getUserAgent().isPresent()) { - String product_name = t.getUserAgent().get().getProductName(); - String product_version = t.getUserAgent().get().getProductVersion(); + if (t.getUserAgentEntry() != null) { + String product_name = t.getUserAgentEntry().getProductName(); + String product_version = t.getUserAgentEntry().getProductVersion(); props.put("UserAgentEntry", product_name + "/" + product_version); } diff --git a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java index 9665e43..15f7c39 100644 --- a/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java +++ b/src/main/java/org/embulk/output/databricks/DatabricksAPIClient.java @@ -18,8 +18,8 @@ public static DatabricksAPIClient create(DatabricksOutputPlugin.DatabricksPlugin } private static void setUserAgent(DatabricksOutputPlugin.DatabricksPluginTask task) { - String name = task.getUserAgent().get().getProductName(); - String version = task.getUserAgent().get().getProductVersion(); + String name = task.getUserAgentEntry().getProductName(); + String version = task.getUserAgentEntry().getProductVersion(); UserAgent.withProduct(name, version); } From 6f43a2dfde58b5dfed9891fa66a7b40504cb043c Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 14 Aug 2024 04:40:30 +0000 Subject: [PATCH 6/9] fix condition --- .../embulk/output/DatabricksOutputPlugin.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index 65ec4cb..bc34fbc 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -57,16 +57,19 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") public UserAgentEntry getUserAgentEntry(); + } - public interface UserAgentEntry extends Task { - @Config("product_name") - @ConfigDefault("\"unknown\"") - public String getProductName(); + public interface UserAgentEntry extends Task { + String defaultProductName = "unknown"; + String defaultProductVersion = "0.0.0"; - @Config("product_version") - @ConfigDefault("\"0.0.0\"") - public String getProductVersion(); - } + @Config("product_name") + @ConfigDefault("\"unknown\"") + public String getProductName(); + + @Config("product_version") + @ConfigDefault("\"0.0.0\"") + public String getProductVersion(); } @Override @@ -107,12 +110,13 @@ protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMet props.put("ConnCatalog", t.getCatalogName()); props.put("ConnSchema", t.getSchemaName()); props.putAll(t.getOptions()); - // overwrite UserAgentEntry property if the same property is set in options - if (t.getUserAgentEntry() != null) { - String product_name = t.getUserAgentEntry().getProductName(); - String product_version = t.getUserAgentEntry().getProductVersion(); - props.put("UserAgentEntry", product_name + "/" + product_version); + String productName = t.getUserAgentEntry().getProductName(); + String productVersion = t.getUserAgentEntry().getProductVersion(); + boolean isSetUserAgentEntryInOptions = props.containsKey("UserAgentEntry"); + // overwrite UserAgentEntry property if the same property is set in options + if (isSetUserAgentEntry(isSetUserAgentEntryInOptions, productName, productVersion)) { + props.put("UserAgentEntry", productName + "/" + productVersion); } logConnectionProperties(url, props); @@ -254,4 +258,9 @@ public Optional newJdbcSchemaFromTableIfExists( return Optional.of(new JdbcSchema(columns)); } } + + private boolean isSetUserAgentEntry(boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { + boolean isDefaultUserAgent = (productName.equals(UserAgentEntry.defaultProductName) || productVersion.equals(UserAgentEntry.defaultProductVersion)); + return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); + } } From 842f69a051924983ab32aa0d912803e0d2ac8404 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Wed, 14 Aug 2024 04:56:41 +0000 Subject: [PATCH 7/9] format --- .../java/org/embulk/output/DatabricksOutputPlugin.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index bc34fbc..394abf6 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -259,8 +259,11 @@ public Optional newJdbcSchemaFromTableIfExists( } } - private boolean isSetUserAgentEntry(boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { - boolean isDefaultUserAgent = (productName.equals(UserAgentEntry.defaultProductName) || productVersion.equals(UserAgentEntry.defaultProductVersion)); + private boolean isSetUserAgentEntry( + boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { + boolean isDefaultUserAgent = + (productName.equals(UserAgentEntry.defaultProductName) + || productVersion.equals(UserAgentEntry.defaultProductVersion)); return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); } } From b9eb3f4fbd2c8e81b003d659f9de7322e20eb2f7 Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 15 Aug 2024 02:35:41 +0000 Subject: [PATCH 8/9] remove condition --- .../embulk/output/DatabricksOutputPlugin.java | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index 394abf6..73e5bb9 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -57,19 +57,16 @@ public interface DatabricksPluginTask extends PluginTask { @Config("user_agent") @ConfigDefault("{}") public UserAgentEntry getUserAgentEntry(); - } - - public interface UserAgentEntry extends Task { - String defaultProductName = "unknown"; - String defaultProductVersion = "0.0.0"; - - @Config("product_name") - @ConfigDefault("\"unknown\"") - public String getProductName(); - @Config("product_version") - @ConfigDefault("\"0.0.0\"") - public String getProductVersion(); + public interface UserAgentEntry extends Task { + @Config("product_name") + @ConfigDefault("\"unknown\"") + public String getProductName(); + + @Config("product_version") + @ConfigDefault("\"0.0.0\"") + public String getProductVersion(); + } } @Override @@ -111,13 +108,10 @@ protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMet props.put("ConnSchema", t.getSchemaName()); props.putAll(t.getOptions()); + // overwrite UserAgentEntry property if the same property is set in options String productName = t.getUserAgentEntry().getProductName(); String productVersion = t.getUserAgentEntry().getProductVersion(); - boolean isSetUserAgentEntryInOptions = props.containsKey("UserAgentEntry"); - // overwrite UserAgentEntry property if the same property is set in options - if (isSetUserAgentEntry(isSetUserAgentEntryInOptions, productName, productVersion)) { - props.put("UserAgentEntry", productName + "/" + productVersion); - } + props.put("UserAgentEntry", productName + "/" + productVersion); logConnectionProperties(url, props); return new DatabricksOutputConnector( @@ -258,12 +252,4 @@ public Optional newJdbcSchemaFromTableIfExists( return Optional.of(new JdbcSchema(columns)); } } - - private boolean isSetUserAgentEntry( - boolean isSetUserAgentEntryInOptions, String productName, String productVersion) { - boolean isDefaultUserAgent = - (productName.equals(UserAgentEntry.defaultProductName) - || productVersion.equals(UserAgentEntry.defaultProductVersion)); - return !(isSetUserAgentEntryInOptions && isDefaultUserAgent); - } } From 42564b3e834f6ddbabca14b75097081eb96c4a2b Mon Sep 17 00:00:00 2001 From: yuki-tashiro <43487435+yu-kioo@users.noreply.github.com> Date: Thu, 15 Aug 2024 02:39:13 +0000 Subject: [PATCH 9/9] fix according to linter --- src/main/java/org/embulk/output/DatabricksOutputPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java index 73e5bb9..931c622 100644 --- a/src/main/java/org/embulk/output/DatabricksOutputPlugin.java +++ b/src/main/java/org/embulk/output/DatabricksOutputPlugin.java @@ -62,7 +62,7 @@ public interface UserAgentEntry extends Task { @Config("product_name") @ConfigDefault("\"unknown\"") public String getProductName(); - + @Config("product_version") @ConfigDefault("\"0.0.0\"") public String getProductVersion();