Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

SQOOP-172. Allow passing of connection parameters.

This change introduces a new option that can be used to pass custom
connection parameters while creating JDBC connections. If no connection
parameters are specified, the system defaults to the old behavior.
  • Loading branch information...
commit 20afd9007dee08e4e81026c2c2962568486b052f 1 parent 67b9a7b
Arvind Prabhakar authored
View
4 src/docs/man/common-args.txt
@@ -29,6 +29,9 @@ Database connection and common options
--driver (class-name)::
Manually specify JDBC driver class to use
+--connection-param-file (filename)::
+ Optional properties file that provides connection parameters
+
--hadoop-home (dir)::
Override $HADOOP_HOME
@@ -47,4 +50,3 @@ Database connection and common options
--verbose::
Print more information while working
-
View
2  src/docs/user/common-args.txt
@@ -32,5 +32,7 @@ Argument Description
+\--password <password>+ Set authentication password
+\--username <username>+ Set authentication username
+\--verbose+ Print more information while working
++\--connection-param-file <filename>+ Optional properties file that\
+ provides connection parameters
-------------------------------------------------------------------------------
View
9 src/docs/user/connecting.txt
@@ -84,4 +84,11 @@ $ sqoop import --driver com.microsoft.jdbc.sqlserver.SQLServerDriver \
--connect <connect-string> ...
----
-
+When connecting to a database using JDBC, you can optionally specify extra
+JDBC parameters via a property file using the option
++\--connection-param-file+. The contents of this file are parsed as standard
+Java properties and passed into the driver while creating a connection.
+
+NOTE: The parameters specified via the optional property file are only
+applicable to JDBC connections. Any fastpath connectors that use connections
+other than JDBC will ignore these parameters.
View
84 src/java/com/cloudera/sqoop/SqoopOptions.java
@@ -23,6 +23,7 @@
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
@@ -32,7 +33,6 @@
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.lib.LargeObjectLoader;
-
import com.cloudera.sqoop.tool.SqoopTool;
import com.cloudera.sqoop.util.RandomHash;
import com.cloudera.sqoop.util.StoredAsProperty;
@@ -113,6 +113,7 @@ public String toString() {
@StoredAsProperty("db.username") private String username;
@StoredAsProperty("db.export.staging.table") private String stagingTableName;
@StoredAsProperty("db.clear.staging.table") private boolean clearStagingTable;
+ private Properties connectionParams; //Properties stored as db.connect.params
// May not be serialized, based on configuration.
@@ -419,6 +420,69 @@ private void setArgArrayProperties(Properties props, String prefix,
}
}
+ /**
+ * This method encodes the property key values found in the provided
+ * properties instance <tt>values</tt> into another properties instance
+ * <tt>props</tt>. The specified <tt>prefix</tt> is used as a namespace
+ * qualifier for keys when inserting. This allows easy introspection of the
+ * property key values in <tt>props</tt> instance to later separate out all
+ * the properties that belong to the <tt>values</tt> instance.
+ * @param props the container properties instance
+ * @param prefix the prefix for qualifying contained property keys.
+ * @param values the contained properties instance, all of whose elements will
+ * be added to the container properties instance.
+ *
+ * @see #getPropertiesAsNetstedProperties(Properties, String)
+ */
+ private void setPropertiesAsNestedProperties(Properties props,
+ String prefix, Properties values) {
+ String nestedPropertyPrefix = prefix + ".";
+ if (null == values || values.size() == 0) {
+ Iterator<String> it = props.stringPropertyNames().iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ if (name.startsWith(nestedPropertyPrefix)) {
+ props.remove(name);
+ }
+ }
+ } else {
+ Iterator<String> it = values.stringPropertyNames().iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ putProperty(props,
+ nestedPropertyPrefix + name, values.getProperty(name));
+ }
+ }
+ }
+
+ /**
+ * This method decodes the property key values found in the provided
+ * properties instance <tt>props</tt> that have keys beginning with the
+ * given prefix. Matching elements from this properties instance are modified
+ * so that their prefix is dropped.
+ * @param props the properties container
+ * @param prefix the prefix qualifying properties that need to be removed
+ * @return a new properties instance that contains all matching elements from
+ * the container properties.
+ */
+ private Properties getPropertiesAsNetstedProperties(
+ Properties props, String prefix) {
+ Properties nestedProps = new Properties();
+ String nestedPropertyPrefix = prefix + ".";
+ int index = nestedPropertyPrefix.length();
+ if (props != null && props.size() > 0) {
+ Iterator<String> it = props.stringPropertyNames().iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ if (name.startsWith(nestedPropertyPrefix)){
+ String shortName = name.substring(index);
+ nestedProps.put(shortName, props.get(name));
+ }
+ }
+ }
+ return nestedProps;
+ }
+
@SuppressWarnings("unchecked")
/**
* Given a set of properties, load this into the current SqoopOptions
@@ -496,6 +560,9 @@ public void loadProperties(Properties props) {
this.extraArgs = getArgArrayProperty(props, "tool.arguments",
this.extraArgs);
+ this.connectionParams =
+ getPropertiesAsNetstedProperties(props, "db.connect.params");
+
// Delimiters were previously memoized; don't let the tool override
// them with defaults.
this.areDelimsManuallySet = true;
@@ -565,6 +632,9 @@ public Properties writeProperties() {
this.outputDelimiters);
setArgArrayProperties(props, "tool.arguments", this.extraArgs);
+ setPropertiesAsNestedProperties(props,
+ "db.connect.params", this.connectionParams);
+
return props;
}
@@ -596,6 +666,10 @@ public Object clone() {
other.extraArgs = Arrays.copyOf(extraArgs, extraArgs.length);
}
+ if (null != connectionParams) {
+ other.setConnectionParams(this.connectionParams);
+ }
+
return other;
} catch (CloneNotSupportedException cnse) {
// Shouldn't happen.
@@ -1755,5 +1829,13 @@ public String getInNullNonStringValue() {
return inNullNonStringValue;
}
+ public void setConnectionParams(Properties params) {
+ connectionParams = new Properties();
+ connectionParams.putAll(params);
+ }
+
+ public Properties getConnectionParams() {
+ return connectionParams;
+ }
}
View
31 src/java/com/cloudera/sqoop/manager/OracleManager.java
@@ -31,6 +31,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -268,12 +269,32 @@ protected Connection makeConnection() throws SQLException {
if (null == connection) {
// Couldn't pull one from the cache. Get a new one.
LOG.debug("Creating a new connection for "
- + connectStr + "/" + username);
- if (null == username) {
- connection = DriverManager.getConnection(connectStr);
+ + connectStr + ", using username: " + username);
+ Properties connectionParams = options.getConnectionParams();
+ if (connectionParams != null && connectionParams.size() > 0) {
+ LOG.debug("User specified connection params. "
+ + "Using properties specific API for making connection.");
+
+ Properties props = new Properties();
+ if (username != null) {
+ props.put("user", username);
+ }
+
+ if (password != null) {
+ props.put("password", password);
+ }
+
+ props.putAll(connectionParams);
+ connection = DriverManager.getConnection(connectStr, props);
} else {
- connection = DriverManager.getConnection(connectStr, username,
- password);
+ LOG.debug("No connection paramenters specified. "
+ + "Using regular API for making connection.");
+ if (username == null) {
+ connection = DriverManager.getConnection(connectStr);
+ } else {
+ connection = DriverManager.getConnection(
+ connectStr, username, password);
+ }
}
}
View
32 src/java/com/cloudera/sqoop/manager/SqlManager.java
@@ -48,6 +48,7 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -191,7 +192,7 @@ protected String getColTypesQuery(String tableName) {
try {
results = execute(stmt);
} catch (SQLException sqlE) {
- LOG.error("Error executing statement: " + sqlE.toString());
+ LOG.error("Error executing statement: " + sqlE.toString(), sqlE);
release();
return null;
}
@@ -637,11 +638,32 @@ protected Connection makeConnection() throws SQLException {
String username = options.getUsername();
String password = options.getPassword();
- if (null == username) {
- connection = DriverManager.getConnection(options.getConnectString());
+ String connectString = options.getConnectString();
+ Properties connectionParams = options.getConnectionParams();
+ if (connectionParams != null && connectionParams.size() > 0) {
+ LOG.debug("User specified connection params. "
+ + "Using properties specific API for making connection.");
+
+ Properties props = new Properties();
+ if (username != null) {
+ props.put("user", username);
+ }
+
+ if (password != null) {
+ props.put("password", password);
+ }
+
+ props.putAll(connectionParams);
+ connection = DriverManager.getConnection(connectString, props);
} else {
- connection = DriverManager.getConnection(options.getConnectString(),
- username, password);
+ LOG.debug("No connection paramenters specified. "
+ + "Using regular API for making connection.");
+ if (username == null) {
+ connection = DriverManager.getConnection(connectString);
+ } else {
+ connection = DriverManager.getConnection(
+ connectString, username, password);
+ }
}
// We only use this for metadata queries. Loosest semantics are okay.
View
46 src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
@@ -18,8 +18,13 @@
package com.cloudera.sqoop.tool;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.sql.SQLException;
import java.util.Arrays;
+import java.util.Properties;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
@@ -63,6 +68,7 @@
public static final String CONNECT_STRING_ARG = "connect";
public static final String CONN_MANAGER_CLASS_NAME =
"connection-manager";
+ public static final String CONNECT_PARAM_FILE = "connection-param-file";
public static final String DRIVER_ARG = "driver";
public static final String USERNAME_ARG = "username";
public static final String PASSWORD_ARG = "password";
@@ -341,9 +347,13 @@ protected RelatedOptions getCommonOptions() {
.withLongOpt(CONNECT_STRING_ARG)
.create());
commonOpts.addOption(OptionBuilder.withArgName("class-name")
- .hasArg().withDescription("Specify connection manager class name")
- .withLongOpt(CONN_MANAGER_CLASS_NAME)
- .create());
+ .hasArg().withDescription("Specify connection manager class name")
+ .withLongOpt(CONN_MANAGER_CLASS_NAME)
+ .create());
+ commonOpts.addOption(OptionBuilder.withArgName("properties-file")
+ .hasArg().withDescription("Specify connection parameters file")
+ .withLongOpt(CONNECT_PARAM_FILE)
+ .create());
commonOpts.addOption(OptionBuilder.withArgName("class-name")
.hasArg().withDescription("Manually specify JDBC driver class to use")
.withLongOpt(DRIVER_ARG)
@@ -616,6 +626,36 @@ protected void applyCommonOptions(CommandLine in, SqoopOptions out)
out.setConnManagerClassName(in.getOptionValue(CONN_MANAGER_CLASS_NAME));
}
+ if (in.hasOption(CONNECT_PARAM_FILE)) {
+ File paramFile = new File(in.getOptionValue(CONNECT_PARAM_FILE));
+ if (!paramFile.exists()) {
+ throw new InvalidOptionsException(
+ "Specified connection parameter file not found: " + paramFile);
+ }
+ InputStream inStream = null;
+ Properties connectionParams = new Properties();
+ try {
+ inStream = new FileInputStream(
+ new File(in.getOptionValue(CONNECT_PARAM_FILE)));
+ connectionParams.load(inStream);
+ } catch (IOException ex) {
+ LOG.warn("Failed to load connection parameter file", ex);
+ throw new InvalidOptionsException(
+ "Error while loading connection parameter file: "
+ + ex.getMessage());
+ } finally {
+ if (inStream != null) {
+ try {
+ inStream.close();
+ } catch (IOException ex) {
+ LOG.warn("Failed to close input stream", ex);
+ }
+ }
+ }
+ LOG.debug("Loaded connection parameters: " + connectionParams);
+ out.setConnectionParams(connectionParams);
+ }
+
if (in.hasOption(NULL_STRING)) {
out.setNullStringValue(in.getOptionValue(NULL_STRING));
}
View
26 src/test/com/cloudera/sqoop/TestSqoopOptions.java
@@ -263,6 +263,14 @@ public void testPropertySerialization1() {
out.setHiveImport(true);
out.setFetchSize(null);
+ Properties connParams = new Properties();
+ connParams.put("conn.timeout", "3000");
+ connParams.put("conn.buffer_size", "256");
+ connParams.put("conn.dummy", "dummy");
+ connParams.put("conn.foo", "bar");
+
+ out.setConnectionParams(connParams);
+
Properties outProps = out.writeProperties();
SqoopOptions in = new SqoopOptions();
@@ -271,6 +279,11 @@ public void testPropertySerialization1() {
Properties inProps = in.writeProperties();
assertEquals("properties don't match", outProps, inProps);
+
+ assertEquals("connection params don't match",
+ connParams, out.getConnectionParams());
+ assertEquals("connection params don't match",
+ connParams, in.getConnectionParams());
}
public void testPropertySerialization2() {
@@ -290,6 +303,15 @@ public void testPropertySerialization2() {
out.setHiveImport(true);
out.setFetchSize(42);
+ Properties connParams = new Properties();
+ connParams.setProperty("a", "value-a");
+ connParams.setProperty("b", "value-b");
+ connParams.setProperty("a.b", "value-a.b");
+ connParams.setProperty("a.b.c", "value-a.b.c");
+ connParams.setProperty("aaaaaaaaaa.bbbbbbb.cccccccc", "value-abc");
+
+ out.setConnectionParams(connParams);
+
Properties outProps = out.writeProperties();
SqoopOptions in = new SqoopOptions();
@@ -298,6 +320,10 @@ public void testPropertySerialization2() {
Properties inProps = in.writeProperties();
assertEquals("properties don't match", outProps, inProps);
+ assertEquals("connection params don't match",
+ connParams, out.getConnectionParams());
+ assertEquals("connection params don't match",
+ connParams, in.getConnectionParams());
}
}
View
8 src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java
@@ -26,13 +26,7 @@
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.TimeZone;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -176,7 +170,7 @@ public void testSQLServerImport() throws IOException {
String [] expectedResults = {
"1,Aaron,1000000.0,engineering",
"2,Bob,400.0,sales",
- "3,Fred,15.0,marketing"
+ "3,Fred,15.0,marketing",
};
runSQLServerTest(expectedResults);
Please sign in to comment.
Something went wrong with that request. Please try again.