Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

SQOOP-172. Allow passing of connection parameters.

This change introduces a new option that can be used to pass custom
connection parameters while creating JDBC connections. If no connection
parameters are specified, the system defaults to the old behavior.
  • Loading branch information...
commit 20afd9007dee08e4e81026c2c2962568486b052f 1 parent 67b9a7b
Arvind Prabhakar authored July 05, 2011
4  src/docs/man/common-args.txt
@@ -29,6 +29,9 @@ Database connection and common options
29 29
 --driver (class-name)::
30 30
   Manually specify JDBC driver class to use
31 31
 
  32
+--connection-param-file (filename)::
  33
+  Optional properties file that provides connection parameters
  34
+
32 35
 --hadoop-home (dir)::
33 36
   Override $HADOOP_HOME
34 37
 
@@ -47,4 +50,3 @@ Database connection and common options
47 50
 
48 51
 --verbose::
49 52
   Print more information while working
50  
-
2  src/docs/user/common-args.txt
@@ -32,5 +32,7 @@ Argument                                  Description
32 32
 +\--password <password>+                  Set authentication password
33 33
 +\--username <username>+                  Set authentication username
34 34
 +\--verbose+                              Print more information while working
  35
++\--connection-param-file <filename>+     Optional properties file that\
  36
+                                          provides connection parameters
35 37
 -------------------------------------------------------------------------------
36 38
 
9  src/docs/user/connecting.txt
@@ -84,4 +84,11 @@ $ sqoop import --driver com.microsoft.jdbc.sqlserver.SQLServerDriver \
84 84
     --connect <connect-string> ...
85 85
 ----
86 86
 
87  
-
  87
+When connecting to a database using JDBC, you can optionally specify extra
  88
+JDBC parameters via a property file using the option
  89
++\--connection-param-file+. The contents of this file are parsed as standard
  90
+Java properties and passed into the driver while creating a connection.
  91
+
  92
+NOTE: The parameters specified via the optional property file are only
  93
+applicable to JDBC connections. Any fastpath connectors that use connections
  94
+other than JDBC will ignore these parameters.
84  src/java/com/cloudera/sqoop/SqoopOptions.java
@@ -23,6 +23,7 @@
23 23
 import java.lang.reflect.Field;
24 24
 import java.util.ArrayList;
25 25
 import java.util.Arrays;
  26
+import java.util.Iterator;
26 27
 import java.util.Map;
27 28
 import java.util.Properties;
28 29
 
@@ -32,7 +33,6 @@
32 33
 
33 34
 import com.cloudera.sqoop.lib.DelimiterSet;
34 35
 import com.cloudera.sqoop.lib.LargeObjectLoader;
35  
-
36 36
 import com.cloudera.sqoop.tool.SqoopTool;
37 37
 import com.cloudera.sqoop.util.RandomHash;
38 38
 import com.cloudera.sqoop.util.StoredAsProperty;
@@ -113,6 +113,7 @@ public String toString() {
113 113
   @StoredAsProperty("db.username") private String username;
114 114
   @StoredAsProperty("db.export.staging.table") private String stagingTableName;
115 115
   @StoredAsProperty("db.clear.staging.table") private boolean clearStagingTable;
  116
+  private Properties connectionParams; //Properties stored as db.connect.params
116 117
 
117 118
 
118 119
   // May not be serialized, based on configuration.
@@ -419,6 +420,69 @@ private void setArgArrayProperties(Properties props, String prefix,
419 420
     }
420 421
   }
421 422
 
  423
+  /**
  424
+   * This method encodes the property key values found in the provided
  425
+   * properties instance <tt>values</tt> into another properties instance
  426
+   * <tt>props</tt>. The specified <tt>prefix</tt> is used as a namespace
  427
+   * qualifier for keys when inserting. This allows easy introspection of the
  428
+   * property key values in <tt>props</tt> instance to later separate out all
  429
+   * the properties that belong to the <tt>values</tt> instance.
  430
+   * @param props the container properties instance
  431
+   * @param prefix the prefix for qualifying contained property keys.
  432
+   * @param values the contained properties instance, all of whose elements will
  433
+   *               be added to the container properties instance.
  434
+   *
  435
+   * @see #getPropertiesAsNetstedProperties(Properties, String)
  436
+   */
  437
+  private void setPropertiesAsNestedProperties(Properties props,
  438
+          String prefix, Properties values) {
  439
+    String nestedPropertyPrefix = prefix + ".";
  440
+    if (null == values || values.size() == 0) {
  441
+      Iterator<String> it = props.stringPropertyNames().iterator();
  442
+      while (it.hasNext()) {
  443
+        String name = it.next();
  444
+        if (name.startsWith(nestedPropertyPrefix)) {
  445
+          props.remove(name);
  446
+        }
  447
+      }
  448
+    } else {
  449
+      Iterator<String> it = values.stringPropertyNames().iterator();
  450
+      while (it.hasNext()) {
  451
+        String name = it.next();
  452
+        putProperty(props,
  453
+                nestedPropertyPrefix + name, values.getProperty(name));
  454
+      }
  455
+    }
  456
+  }
  457
+
  458
+  /**
  459
+   * This method decodes the property key values found in the provided
  460
+   * properties instance <tt>props</tt> that have keys beginning with the
  461
+   * given prefix. Matching elements from this properties instance are modified
  462
+   * so that their prefix is dropped.
  463
+   * @param props the properties container
  464
+   * @param prefix the prefix qualifying properties that need to be removed
  465
+   * @return a new properties instance that contains all matching elements from
  466
+   * the container properties.
  467
+   */
  468
+  private Properties getPropertiesAsNetstedProperties(
  469
+          Properties props, String prefix) {
  470
+    Properties nestedProps = new Properties();
  471
+    String nestedPropertyPrefix = prefix + ".";
  472
+    int index = nestedPropertyPrefix.length();
  473
+    if (props != null && props.size() > 0) {
  474
+      Iterator<String> it = props.stringPropertyNames().iterator();
  475
+      while (it.hasNext()) {
  476
+        String name = it.next();
  477
+        if (name.startsWith(nestedPropertyPrefix)){
  478
+          String shortName = name.substring(index);
  479
+          nestedProps.put(shortName, props.get(name));
  480
+        }
  481
+      }
  482
+    }
  483
+    return nestedProps;
  484
+  }
  485
+
422 486
   @SuppressWarnings("unchecked")
423 487
   /**
424 488
    * Given a set of properties, load this into the current SqoopOptions
@@ -496,6 +560,9 @@ public void loadProperties(Properties props) {
496 560
     this.extraArgs = getArgArrayProperty(props, "tool.arguments",
497 561
         this.extraArgs);
498 562
 
  563
+    this.connectionParams =
  564
+        getPropertiesAsNetstedProperties(props, "db.connect.params");
  565
+
499 566
     // Delimiters were previously memoized; don't let the tool override
500 567
     // them with defaults.
501 568
     this.areDelimsManuallySet = true;
@@ -565,6 +632,9 @@ public Properties writeProperties() {
565 632
         this.outputDelimiters);
566 633
     setArgArrayProperties(props, "tool.arguments", this.extraArgs);
567 634
 
  635
+    setPropertiesAsNestedProperties(props,
  636
+            "db.connect.params", this.connectionParams);
  637
+
568 638
     return props;
569 639
   }
570 640
 
@@ -596,6 +666,10 @@ public Object clone() {
596 666
         other.extraArgs = Arrays.copyOf(extraArgs, extraArgs.length);
597 667
       }
598 668
 
  669
+      if (null != connectionParams) {
  670
+        other.setConnectionParams(this.connectionParams);
  671
+      }
  672
+
599 673
       return other;
600 674
     } catch (CloneNotSupportedException cnse) {
601 675
       // Shouldn't happen.
@@ -1755,5 +1829,13 @@ public String getInNullNonStringValue() {
1755 1829
     return inNullNonStringValue;
1756 1830
   }
1757 1831
 
  1832
+  public void setConnectionParams(Properties params) {
  1833
+    connectionParams = new Properties();
  1834
+    connectionParams.putAll(params);
  1835
+  }
  1836
+
  1837
+  public Properties getConnectionParams() {
  1838
+    return connectionParams;
  1839
+  }
1758 1840
 }
1759 1841
 
31  src/java/com/cloudera/sqoop/manager/OracleManager.java
@@ -31,6 +31,7 @@
31 31
 import java.util.HashMap;
32 32
 import java.util.List;
33 33
 import java.util.Map;
  34
+import java.util.Properties;
34 35
 
35 36
 import org.apache.commons.logging.Log;
36 37
 import org.apache.commons.logging.LogFactory;
@@ -268,12 +269,32 @@ protected Connection makeConnection() throws SQLException {
268 269
     if (null == connection) {
269 270
       // Couldn't pull one from the cache. Get a new one.
270 271
       LOG.debug("Creating a new connection for "
271  
-          + connectStr + "/" + username);
272  
-      if (null == username) {
273  
-        connection = DriverManager.getConnection(connectStr);
  272
+              + connectStr + ", using username: " + username);
  273
+      Properties connectionParams = options.getConnectionParams();
  274
+      if (connectionParams != null && connectionParams.size() > 0) {
  275
+        LOG.debug("User specified connection params. "
  276
+                  + "Using properties specific API for making connection.");
  277
+
  278
+        Properties props = new Properties();
  279
+        if (username != null) {
  280
+          props.put("user", username);
  281
+        }
  282
+
  283
+        if (password != null) {
  284
+          props.put("password", password);
  285
+        }
  286
+
  287
+        props.putAll(connectionParams);
  288
+        connection = DriverManager.getConnection(connectStr, props);
274 289
       } else {
275  
-        connection = DriverManager.getConnection(connectStr, username,
276  
-            password);
  290
+        LOG.debug("No connection paramenters specified. "
  291
+                + "Using regular API for making connection.");
  292
+        if (username == null) {
  293
+          connection = DriverManager.getConnection(connectStr);
  294
+        } else {
  295
+          connection = DriverManager.getConnection(
  296
+                              connectStr, username, password);
  297
+        }
277 298
       }
278 299
     }
279 300
 
32  src/java/com/cloudera/sqoop/manager/SqlManager.java
@@ -48,6 +48,7 @@
48 48
 import java.util.ArrayList;
49 49
 import java.util.HashMap;
50 50
 import java.util.Map;
  51
+import java.util.Properties;
51 52
 
52 53
 import org.apache.commons.logging.Log;
53 54
 import org.apache.commons.logging.LogFactory;
@@ -191,7 +192,7 @@ protected String getColTypesQuery(String tableName) {
191 192
     try {
192 193
       results = execute(stmt);
193 194
     } catch (SQLException sqlE) {
194  
-      LOG.error("Error executing statement: " + sqlE.toString());
  195
+      LOG.error("Error executing statement: " + sqlE.toString(), sqlE);
195 196
       release();
196 197
       return null;
197 198
     }
@@ -637,11 +638,32 @@ protected Connection makeConnection() throws SQLException {
637 638
 
638 639
     String username = options.getUsername();
639 640
     String password = options.getPassword();
640  
-    if (null == username) {
641  
-      connection = DriverManager.getConnection(options.getConnectString());
  641
+    String connectString = options.getConnectString();
  642
+    Properties connectionParams = options.getConnectionParams();
  643
+    if (connectionParams != null && connectionParams.size() > 0) {
  644
+      LOG.debug("User specified connection params. "
  645
+              + "Using properties specific API for making connection.");
  646
+
  647
+      Properties props = new Properties();
  648
+      if (username != null) {
  649
+        props.put("user", username);
  650
+      }
  651
+
  652
+      if (password != null) {
  653
+        props.put("password", password);
  654
+      }
  655
+
  656
+      props.putAll(connectionParams);
  657
+      connection = DriverManager.getConnection(connectString, props);
642 658
     } else {
643  
-      connection = DriverManager.getConnection(options.getConnectString(),
644  
-          username, password);
  659
+      LOG.debug("No connection paramenters specified. "
  660
+              + "Using regular API for making connection.");
  661
+      if (username == null) {
  662
+        connection = DriverManager.getConnection(connectString);
  663
+      } else {
  664
+        connection = DriverManager.getConnection(
  665
+                        connectString, username, password);
  666
+      }
645 667
     }
646 668
 
647 669
     // We only use this for metadata queries. Loosest semantics are okay.
46  src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
@@ -18,8 +18,13 @@
18 18
 
19 19
 package com.cloudera.sqoop.tool;
20 20
 
  21
+import java.io.File;
  22
+import java.io.FileInputStream;
  23
+import java.io.IOException;
  24
+import java.io.InputStream;
21 25
 import java.sql.SQLException;
22 26
 import java.util.Arrays;
  27
+import java.util.Properties;
23 28
 
24 29
 import org.apache.commons.cli.CommandLine;
25 30
 import org.apache.commons.cli.Option;
@@ -63,6 +68,7 @@
63 68
   public static final String CONNECT_STRING_ARG = "connect";
64 69
   public static final String CONN_MANAGER_CLASS_NAME =
65 70
       "connection-manager";
  71
+  public static final String CONNECT_PARAM_FILE = "connection-param-file";
66 72
   public static final String DRIVER_ARG = "driver";
67 73
   public static final String USERNAME_ARG = "username";
68 74
   public static final String PASSWORD_ARG = "password";
@@ -341,9 +347,13 @@ protected RelatedOptions getCommonOptions() {
341 347
         .withLongOpt(CONNECT_STRING_ARG)
342 348
         .create());
343 349
     commonOpts.addOption(OptionBuilder.withArgName("class-name")
344  
-            .hasArg().withDescription("Specify connection manager class name")
345  
-            .withLongOpt(CONN_MANAGER_CLASS_NAME)
346  
-            .create());
  350
+        .hasArg().withDescription("Specify connection manager class name")
  351
+        .withLongOpt(CONN_MANAGER_CLASS_NAME)
  352
+        .create());
  353
+    commonOpts.addOption(OptionBuilder.withArgName("properties-file")
  354
+        .hasArg().withDescription("Specify connection parameters file")
  355
+        .withLongOpt(CONNECT_PARAM_FILE)
  356
+        .create());
347 357
     commonOpts.addOption(OptionBuilder.withArgName("class-name")
348 358
         .hasArg().withDescription("Manually specify JDBC driver class to use")
349 359
         .withLongOpt(DRIVER_ARG)
@@ -616,6 +626,36 @@ protected void applyCommonOptions(CommandLine in, SqoopOptions out)
616 626
         out.setConnManagerClassName(in.getOptionValue(CONN_MANAGER_CLASS_NAME));
617 627
     }
618 628
 
  629
+    if (in.hasOption(CONNECT_PARAM_FILE)) {
  630
+      File paramFile = new File(in.getOptionValue(CONNECT_PARAM_FILE));
  631
+      if (!paramFile.exists()) {
  632
+        throw new InvalidOptionsException(
  633
+                "Specified connection parameter file not found: " + paramFile);
  634
+      }
  635
+      InputStream inStream = null;
  636
+      Properties connectionParams = new Properties();
  637
+      try {
  638
+        inStream = new FileInputStream(
  639
+                      new File(in.getOptionValue(CONNECT_PARAM_FILE)));
  640
+        connectionParams.load(inStream);
  641
+      } catch (IOException ex) {
  642
+        LOG.warn("Failed to load connection parameter file", ex);
  643
+        throw new InvalidOptionsException(
  644
+                "Error while loading connection parameter file: "
  645
+                + ex.getMessage());
  646
+      } finally {
  647
+        if (inStream != null) {
  648
+          try {
  649
+            inStream.close();
  650
+          } catch (IOException ex) {
  651
+            LOG.warn("Failed to close input stream", ex);
  652
+          }
  653
+        }
  654
+      }
  655
+      LOG.debug("Loaded connection parameters: " + connectionParams);
  656
+      out.setConnectionParams(connectionParams);
  657
+    }
  658
+
619 659
     if (in.hasOption(NULL_STRING)) {
620 660
         out.setNullStringValue(in.getOptionValue(NULL_STRING));
621 661
     }
26  src/test/com/cloudera/sqoop/TestSqoopOptions.java
@@ -263,6 +263,14 @@ public void testPropertySerialization1() {
263 263
     out.setHiveImport(true);
264 264
     out.setFetchSize(null);
265 265
 
  266
+    Properties connParams = new Properties();
  267
+    connParams.put("conn.timeout", "3000");
  268
+    connParams.put("conn.buffer_size", "256");
  269
+    connParams.put("conn.dummy", "dummy");
  270
+    connParams.put("conn.foo", "bar");
  271
+
  272
+    out.setConnectionParams(connParams);
  273
+
266 274
     Properties outProps = out.writeProperties();
267 275
 
268 276
     SqoopOptions in = new SqoopOptions();
@@ -271,6 +279,11 @@ public void testPropertySerialization1() {
271 279
     Properties inProps = in.writeProperties();
272 280
 
273 281
     assertEquals("properties don't match", outProps, inProps);
  282
+
  283
+    assertEquals("connection params don't match",
  284
+            connParams, out.getConnectionParams());
  285
+    assertEquals("connection params don't match",
  286
+            connParams, in.getConnectionParams());
274 287
   }
275 288
 
276 289
   public void testPropertySerialization2() {
@@ -290,6 +303,15 @@ public void testPropertySerialization2() {
290 303
     out.setHiveImport(true);
291 304
     out.setFetchSize(42);
292 305
 
  306
+    Properties connParams = new Properties();
  307
+    connParams.setProperty("a", "value-a");
  308
+    connParams.setProperty("b", "value-b");
  309
+    connParams.setProperty("a.b", "value-a.b");
  310
+    connParams.setProperty("a.b.c", "value-a.b.c");
  311
+    connParams.setProperty("aaaaaaaaaa.bbbbbbb.cccccccc", "value-abc");
  312
+
  313
+    out.setConnectionParams(connParams);
  314
+
293 315
     Properties outProps = out.writeProperties();
294 316
 
295 317
     SqoopOptions in = new SqoopOptions();
@@ -298,6 +320,10 @@ public void testPropertySerialization2() {
298 320
     Properties inProps = in.writeProperties();
299 321
 
300 322
     assertEquals("properties don't match", outProps, inProps);
  323
+    assertEquals("connection params don't match",
  324
+            connParams, out.getConnectionParams());
  325
+    assertEquals("connection params don't match",
  326
+            connParams, in.getConnectionParams());
301 327
   }
302 328
 
303 329
 }
8  src/test/com/cloudera/sqoop/manager/SQLServerManagerImportManualTest.java
@@ -26,13 +26,7 @@
26 26
 import java.sql.Connection;
27 27
 import java.sql.SQLException;
28 28
 import java.sql.Statement;
29  
-import java.text.DateFormat;
30  
-import java.text.ParseException;
31  
-import java.text.SimpleDateFormat;
32 29
 import java.util.ArrayList;
33  
-import java.util.Calendar;
34  
-import java.util.Date;
35  
-import java.util.TimeZone;
36 30
 
37 31
 import org.apache.commons.logging.Log;
38 32
 import org.apache.commons.logging.LogFactory;
@@ -176,7 +170,7 @@ public void testSQLServerImport() throws IOException {
176 170
     String [] expectedResults = {
177 171
       "1,Aaron,1000000.0,engineering",
178 172
       "2,Bob,400.0,sales",
179  
-      "3,Fred,15.0,marketing"
  173
+      "3,Fred,15.0,marketing",
180 174
     };
181 175
 
182 176
     runSQLServerTest(expectedResults);

0 notes on commit 20afd90

Please sign in to comment.
Something went wrong with that request. Please try again.