Permalink
Browse files

SQOOP-172. Allow passing of connection parameters.

This change introduces a new option that can be used to pass custom
connection parameters while creating JDBC connections. If no connection
parameters are specified, the system defaults to the old behavior.
  • Loading branch information...
Arvind Prabhakar
Arvind Prabhakar committed Jul 5, 2011
1 parent 67b9a7b commit 20afd9007dee08e4e81026c2c2962568486b052f
@@ -29,6 +29,9 @@ Database connection and common options
--driver (class-name)::
Manually specify JDBC driver class to use
+--connection-param-file (filename)::
+ Optional properties file that provides connection parameters
+
--hadoop-home (dir)::
Override $HADOOP_HOME
@@ -47,4 +50,3 @@ Database connection and common options
--verbose::
Print more information while working
-
@@ -32,5 +32,7 @@ Argument Description
+\--password <password>+ Set authentication password
+\--username <username>+ Set authentication username
+\--verbose+ Print more information while working
++\--connection-param-file <filename>+ Optional properties file that\
+ provides connection parameters
-------------------------------------------------------------------------------
@@ -84,4 +84,11 @@ $ sqoop import --driver com.microsoft.jdbc.sqlserver.SQLServerDriver \
--connect <connect-string> ...
----
-
+When connecting to a database using JDBC, you can optionally specify extra
+JDBC parameters via a property file using the option
++\--connection-param-file+. The contents of this file are parsed as standard
+Java properties and passed into the driver while creating a connection.
+
+NOTE: The parameters specified via the optional property file are only
+applicable to JDBC connections. Any fastpath connectors that use connections
+other than JDBC will ignore these parameters.
@@ -23,6 +23,7 @@
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
@@ -32,7 +33,6 @@
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.lib.LargeObjectLoader;
-
import com.cloudera.sqoop.tool.SqoopTool;
import com.cloudera.sqoop.util.RandomHash;
import com.cloudera.sqoop.util.StoredAsProperty;
@@ -113,6 +113,7 @@ public String toString() {
@StoredAsProperty("db.username") private String username;
@StoredAsProperty("db.export.staging.table") private String stagingTableName;
@StoredAsProperty("db.clear.staging.table") private boolean clearStagingTable;
+ private Properties connectionParams; //Properties stored as db.connect.params
// May not be serialized, based on configuration.
@@ -419,6 +420,69 @@ private void setArgArrayProperties(Properties props, String prefix,
}
}
+ /**
+ * This method encodes the property key values found in the provided
+ * properties instance <tt>values</tt> into another properties instance
+ * <tt>props</tt>. The specified <tt>prefix</tt> is used as a namespace
+ * qualifier for keys when inserting. This allows easy introspection of the
+ * property key values in <tt>props</tt> instance to later separate out all
+ * the properties that belong to the <tt>values</tt> instance.
+ * @param props the container properties instance
+ * @param prefix the prefix for qualifying contained property keys.
+ * @param values the contained properties instance, all of whose elements will
+ * be added to the container properties instance.
+ *
+ * @see #getPropertiesAsNetstedProperties(Properties, String)
+ */
+ private void setPropertiesAsNestedProperties(Properties props,
+ String prefix, Properties values) {
+ String nestedPropertyPrefix = prefix + ".";
+ if (null == values || values.size() == 0) {
+ Iterator<String> it = props.stringPropertyNames().iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ if (name.startsWith(nestedPropertyPrefix)) {
+ props.remove(name);
+ }
+ }
+ } else {
+ Iterator<String> it = values.stringPropertyNames().iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ putProperty(props,
+ nestedPropertyPrefix + name, values.getProperty(name));
+ }
+ }
+ }
+
+ /**
+ * This method decodes the property key values found in the provided
+ * properties instance <tt>props</tt> that have keys beginning with the
+ * given prefix. Matching elements from this properties instance are modified
+ * so that their prefix is dropped.
+ * @param props the properties container
+ * @param prefix the prefix qualifying properties that need to be removed
+ * @return a new properties instance that contains all matching elements from
+ * the container properties.
+ */
+ private Properties getPropertiesAsNetstedProperties(
+ Properties props, String prefix) {
+ Properties nestedProps = new Properties();
+ String nestedPropertyPrefix = prefix + ".";
+ int index = nestedPropertyPrefix.length();
+ if (props != null && props.size() > 0) {
+ Iterator<String> it = props.stringPropertyNames().iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ if (name.startsWith(nestedPropertyPrefix)){
+ String shortName = name.substring(index);
+ nestedProps.put(shortName, props.get(name));
+ }
+ }
+ }
+ return nestedProps;
+ }
+
@SuppressWarnings("unchecked")
/**
* Given a set of properties, load this into the current SqoopOptions
@@ -496,6 +560,9 @@ public void loadProperties(Properties props) {
this.extraArgs = getArgArrayProperty(props, "tool.arguments",
this.extraArgs);
+ this.connectionParams =
+ getPropertiesAsNetstedProperties(props, "db.connect.params");
+
// Delimiters were previously memoized; don't let the tool override
// them with defaults.
this.areDelimsManuallySet = true;
@@ -565,6 +632,9 @@ public Properties writeProperties() {
this.outputDelimiters);
setArgArrayProperties(props, "tool.arguments", this.extraArgs);
+ setPropertiesAsNestedProperties(props,
+ "db.connect.params", this.connectionParams);
+
return props;
}
@@ -596,6 +666,10 @@ public Object clone() {
other.extraArgs = Arrays.copyOf(extraArgs, extraArgs.length);
}
+ if (null != connectionParams) {
+ other.setConnectionParams(this.connectionParams);
+ }
+
return other;
} catch (CloneNotSupportedException cnse) {
// Shouldn't happen.
@@ -1755,5 +1829,13 @@ public String getInNullNonStringValue() {
return inNullNonStringValue;
}
+ public void setConnectionParams(Properties params) {
+ connectionParams = new Properties();
+ connectionParams.putAll(params);
+ }
+
+ public Properties getConnectionParams() {
+ return connectionParams;
+ }
}
@@ -31,6 +31,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -268,12 +269,32 @@ protected Connection makeConnection() throws SQLException {
if (null == connection) {
// Couldn't pull one from the cache. Get a new one.
LOG.debug("Creating a new connection for "
- + connectStr + "/" + username);
- if (null == username) {
- connection = DriverManager.getConnection(connectStr);
+ + connectStr + ", using username: " + username);
+ Properties connectionParams = options.getConnectionParams();
+ if (connectionParams != null && connectionParams.size() > 0) {
+ LOG.debug("User specified connection params. "
+ + "Using properties specific API for making connection.");
+
+ Properties props = new Properties();
+ if (username != null) {
+ props.put("user", username);
+ }
+
+ if (password != null) {
+ props.put("password", password);
+ }
+
+ props.putAll(connectionParams);
+ connection = DriverManager.getConnection(connectStr, props);
} else {
- connection = DriverManager.getConnection(connectStr, username,
- password);
+ LOG.debug("No connection paramenters specified. "
+ + "Using regular API for making connection.");
+ if (username == null) {
+ connection = DriverManager.getConnection(connectStr);
+ } else {
+ connection = DriverManager.getConnection(
+ connectStr, username, password);
+ }
}
}
@@ -48,6 +48,7 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -191,7 +192,7 @@ protected String getColTypesQuery(String tableName) {
try {
results = execute(stmt);
} catch (SQLException sqlE) {
- LOG.error("Error executing statement: " + sqlE.toString());
+ LOG.error("Error executing statement: " + sqlE.toString(), sqlE);
release();
return null;
}
@@ -637,11 +638,32 @@ protected Connection makeConnection() throws SQLException {
String username = options.getUsername();
String password = options.getPassword();
- if (null == username) {
- connection = DriverManager.getConnection(options.getConnectString());
+ String connectString = options.getConnectString();
+ Properties connectionParams = options.getConnectionParams();
+ if (connectionParams != null && connectionParams.size() > 0) {
+ LOG.debug("User specified connection params. "
+ + "Using properties specific API for making connection.");
+
+ Properties props = new Properties();
+ if (username != null) {
+ props.put("user", username);
+ }
+
+ if (password != null) {
+ props.put("password", password);
+ }
+
+ props.putAll(connectionParams);
+ connection = DriverManager.getConnection(connectString, props);
} else {
- connection = DriverManager.getConnection(options.getConnectString(),
- username, password);
+ LOG.debug("No connection paramenters specified. "
+ + "Using regular API for making connection.");
+ if (username == null) {
+ connection = DriverManager.getConnection(connectString);
+ } else {
+ connection = DriverManager.getConnection(
+ connectString, username, password);
+ }
}
// We only use this for metadata queries. Loosest semantics are okay.
@@ -18,8 +18,13 @@
package com.cloudera.sqoop.tool;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.sql.SQLException;
import java.util.Arrays;
+import java.util.Properties;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
@@ -63,6 +68,7 @@
public static final String CONNECT_STRING_ARG = "connect";
public static final String CONN_MANAGER_CLASS_NAME =
"connection-manager";
+ public static final String CONNECT_PARAM_FILE = "connection-param-file";
public static final String DRIVER_ARG = "driver";
public static final String USERNAME_ARG = "username";
public static final String PASSWORD_ARG = "password";
@@ -341,9 +347,13 @@ protected RelatedOptions getCommonOptions() {
.withLongOpt(CONNECT_STRING_ARG)
.create());
commonOpts.addOption(OptionBuilder.withArgName("class-name")
- .hasArg().withDescription("Specify connection manager class name")
- .withLongOpt(CONN_MANAGER_CLASS_NAME)
- .create());
+ .hasArg().withDescription("Specify connection manager class name")
+ .withLongOpt(CONN_MANAGER_CLASS_NAME)
+ .create());
+ commonOpts.addOption(OptionBuilder.withArgName("properties-file")
+ .hasArg().withDescription("Specify connection parameters file")
+ .withLongOpt(CONNECT_PARAM_FILE)
+ .create());
commonOpts.addOption(OptionBuilder.withArgName("class-name")
.hasArg().withDescription("Manually specify JDBC driver class to use")
.withLongOpt(DRIVER_ARG)
@@ -616,6 +626,36 @@ protected void applyCommonOptions(CommandLine in, SqoopOptions out)
out.setConnManagerClassName(in.getOptionValue(CONN_MANAGER_CLASS_NAME));
}
+ if (in.hasOption(CONNECT_PARAM_FILE)) {
+ File paramFile = new File(in.getOptionValue(CONNECT_PARAM_FILE));
+ if (!paramFile.exists()) {
+ throw new InvalidOptionsException(
+ "Specified connection parameter file not found: " + paramFile);
+ }
+ InputStream inStream = null;
+ Properties connectionParams = new Properties();
+ try {
+ inStream = new FileInputStream(
+ new File(in.getOptionValue(CONNECT_PARAM_FILE)));
+ connectionParams.load(inStream);
+ } catch (IOException ex) {
+ LOG.warn("Failed to load connection parameter file", ex);
+ throw new InvalidOptionsException(
+ "Error while loading connection parameter file: "
+ + ex.getMessage());
+ } finally {
+ if (inStream != null) {
+ try {
+ inStream.close();
+ } catch (IOException ex) {
+ LOG.warn("Failed to close input stream", ex);
+ }
+ }
+ }
+ LOG.debug("Loaded connection parameters: " + connectionParams);
+ out.setConnectionParams(connectionParams);
+ }
+
if (in.hasOption(NULL_STRING)) {
out.setNullStringValue(in.getOptionValue(NULL_STRING));
}
Oops, something went wrong.

0 comments on commit 20afd90

Please sign in to comment.