HIVE-9664. Hive 'add jar' command should be able to download and add …

…jars from a repository (Anant Nag via cws)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1670246 13f79535-47bb-0310-9956-ffa450edef68

conf/ivysettings.xml

@@ -0,0 +1,37 @@
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+   -->
+
+<!--This file is used by grapes to download dependencies from a maven repository.
+    This is just a template and can be edited to add more repositories.
+-->
+
+<ivysettings>
+  <!--name of the defaultResolver should always be 'downloadGrapes'. -->
+  <settings defaultResolver="downloadGrapes"/>
+  <resolvers>
+    <!-- more resolvers can be added here -->
+    <chain name="downloadGrapes">
+      <!-- This resolver uses ibiblio to find artifacts, compatible with maven2 repository -->
+      <ibiblio name="central" m2compatible="true"/>
+      <!-- File resolver to add jars from the local system. -->
+      <filesystem name="test" checkmodified="true">
+        <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar" />
+      </filesystem>
+    </chain>
+  </resolvers>
+</ivysettings>

itests/pom.xml

@@ -93,6 +93,9 @@
                   mkdir -p $DOWNLOAD_DIR
                   download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
                   cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
+                  sed '/package /d' ${basedir}/${hive.path.to.root}/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java > /tmp/UDFExampleAdd.java
+                  javac -cp  ${settings.localRepository}/org/apache/hive/hive-exec/${project.version}/hive-exec-${project.version}.jar /tmp/UDFExampleAdd.java -d /tmp
+                  jar -cf /tmp/udfexampleadd-1.0.jar -C /tmp UDFExampleAdd.class
                 </echo>
               </target>
             </configuration>

packaging/src/main/assembly/bin.xml

@@ -146,6 +146,7 @@
       <directory>${project.parent.basedir}/conf</directory>
       <includes>
         <include>*.template</include>
+        <include>ivysettings.xml</include>
       </includes>
       <outputDirectory>conf</outputDirectory>
     </fileSet>

pom.xml

@@ -126,6 +126,7 @@
     <!-- httpcomponents are not always in version sync -->
     <httpcomponents.client.version>4.2.5</httpcomponents.client.version>
     <httpcomponents.core.version>4.2.5</httpcomponents.core.version>
+    <ivy.version>2.4.0</ivy.version>
     <jackson.version>1.9.2</jackson.version>
     <javaewah.version>0.3.2</javaewah.version>
     <javolution.version>5.5.1</javolution.version>

ql/pom.xml

@@ -162,6 +162,11 @@
       <artifactId>libfb303</artifactId>
       <version>${libfb303.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ivy</groupId>
+      <artifactId>ivy</artifactId>
+      <version>${ivy.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>

ql/src/java/org/apache/hadoop/hive/ql/session/DependencyResolver.java

@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.session;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import groovy.grape.Grape;
+import groovy.grape.GrapeIvy;
+import groovy.lang.GroovyClassLoader;
+
+
+public class DependencyResolver {
+
+  private static final String HIVE_HOME = "HIVE_HOME";
+  private static final String HIVE_CONF_DIR = "HIVE_CONF_DIR";
+  private String ivysettingsPath;
+  private static LogHelper _console = new LogHelper(LogFactory.getLog("DependencyResolver"));
+
+  public DependencyResolver() {
+
+    // Check if HIVE_CONF_DIR is defined
+    if (System.getenv().containsKey(HIVE_CONF_DIR)) {
+      ivysettingsPath = System.getenv().get(HIVE_CONF_DIR) + "/ivysettings.xml";
+    }
+
+    // If HIVE_CONF_DIR is not defined or file is not found in HIVE_CONF_DIR then check HIVE_HOME/conf
+    if (ivysettingsPath == null || !(new File(ivysettingsPath).exists())) {
+      if (System.getenv().containsKey(HIVE_HOME)) {
+        ivysettingsPath = System.getenv().get(HIVE_HOME) + "/conf/ivysettings.xml";
+      }
+    }
+
+    // If HIVE_HOME is not defined or file is not found in HIVE_HOME/conf then load default ivysettings.xml from class loader
+    if (ivysettingsPath == null || !(new File(ivysettingsPath).exists())) {
+      ivysettingsPath = ClassLoader.getSystemResource("ivysettings.xml").getFile();
+      _console.printInfo("ivysettings.xml file not found in HIVE_HOME or HIVE_CONF_DIR," + ivysettingsPath + " will be used");
+    }
+
+  }
+
+  /**
+   *
+   * @param uri
+   * @return List of URIs of downloaded jars
+   * @throws URISyntaxException
+   * @throws IOException
+   */
+  public List<URI> downloadDependencies(URI uri) throws URISyntaxException, IOException {
+    Map<String, Object> dependencyMap = new HashMap<String, Object>();
+    String authority = uri.getAuthority();
+    if (authority == null) {
+      throw new URISyntaxException(authority, "Invalid url: Expected 'org:module:version', found null");
+    }
+    String[] authorityTokens = authority.toLowerCase().split(":");
+
+    if (authorityTokens.length != 3) {
+      throw new URISyntaxException(authority, "Invalid url: Expected 'org:module:version', found " + authority);
+    }
+
+    dependencyMap.put("org", authorityTokens[0]);
+    dependencyMap.put("module", authorityTokens[1]);
+    dependencyMap.put("version", authorityTokens[2]);
+    Map<String, Object> queryMap = parseQueryString(uri.getQuery());
+    if (queryMap != null) {
+      dependencyMap.putAll(queryMap);
+    }
+    return grab(dependencyMap);
+  }
+
+  /**
+   * @param queryString
+   * @return queryMap Map which contains grape parameters such as transitive, exclude, ext and classifier.
+   * Example: Input:  ext=jar&exclude=org.mortbay.jetty:jetty&transitive=true
+   *          Output:  {[ext]:[jar], [exclude]:{[group]:[org.mortbay.jetty], [module]:[jetty]}, [transitive]:[true]}
+   * @throws URISyntaxException
+   */
+  private Map<String, Object> parseQueryString(String queryString) throws URISyntaxException {
+    if (queryString == null || queryString.isEmpty()) {
+      return null;
+    }
+    List<Map<String, String>> excludeList = new LinkedList<Map<String, String>>();
+    Map<String, Object> queryMap = new HashMap<String, Object>();
+    String[] mapTokens = queryString.split("&");
+    for (String tokens : mapTokens) {
+      String[] mapPair = tokens.split("=");
+      if (mapPair.length != 2) {
+        throw new RuntimeException("Invalid query string: " + queryString);
+      }
+      if (mapPair[0].equals("exclude")) {
+        excludeList.addAll(computeExcludeList(mapPair[1]));
+      } else if (mapPair[0].equals("transitive")) {
+        if (mapPair[1].toLowerCase().equals("true")) {
+          queryMap.put(mapPair[0], true);
+        } else {
+          queryMap.put(mapPair[0], false);
+        }
+      } else {
+        queryMap.put(mapPair[0], mapPair[1]);
+      }
+    }
+    if (!excludeList.isEmpty()) {
+      queryMap.put("exclude", excludeList);
+    }
+    return queryMap;
+  }
+
+  private List<Map<String, String>> computeExcludeList(String excludeString) throws URISyntaxException {
+    String excludes[] = excludeString.split(",");
+    List<Map<String, String>> excludeList = new LinkedList<Map<String, String>>();
+    for (String exclude : excludes) {
+      Map<String, String> tempMap = new HashMap<String, String>();
+      String args[] = exclude.split(":");
+      if (args.length != 2) {
+        throw new URISyntaxException(excludeString,
+            "Invalid exclude string: expected 'org:module,org:module,..', found " + excludeString);
+      }
+      tempMap.put("group", args[0]);
+      tempMap.put("module", args[1]);
+      excludeList.add(tempMap);
+    }
+    return excludeList;
+  }
+
+  /**
+   *
+   * @param dependencies
+   * @return List of URIs of downloaded jars
+   * @throws IOException
+   */
+  private List<URI> grab(Map<String, Object> dependencies) throws IOException {
+    Map<String, Object> args = new HashMap<String, Object>();
+    URI[] localUrls;
+
+    //grape expects excludes key in args map
+    if (dependencies.containsKey("exclude")) {
+      args.put("excludes", dependencies.get("exclude"));
+    }
+
+    //Set transitive to true by default
+    if (!dependencies.containsKey("transitive")) {
+      dependencies.put("transitive", true);
+    }
+
+    args.put("classLoader", new GroovyClassLoader());
+    System.setProperty("grape.config", ivysettingsPath);
+    System.setProperty("groovy.grape.report.downloads", "true");
+    localUrls = Grape.resolve(args, dependencies);
+    if (localUrls == null) {
+      throw new IOException("Not able to download all the dependencies..");
+    }
+    return Arrays.asList(localUrls);
+  }
+}