DependencyResolver.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql.util;

import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.io.File;
import java.io.IOException;

import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.slf4j.LoggerFactory;

import groovy.grape.Grape;
import groovy.lang.GroovyClassLoader;


public class DependencyResolver {

  private static final String HIVE_HOME = "HIVE_HOME";
  private static final String HIVE_CONF_DIR = "HIVE_CONF_DIR";
  private String ivysettingsPath;
  private static LogHelper _console = new LogHelper(LoggerFactory.getLogger("DependencyResolver"));

  public DependencyResolver() {

    // Check if HIVE_CONF_DIR is defined
    if (System.getenv().containsKey(HIVE_CONF_DIR)) {
      ivysettingsPath = System.getenv().get(HIVE_CONF_DIR) + "/ivysettings.xml";
    }

    // If HIVE_CONF_DIR is not defined or file is not found in HIVE_CONF_DIR then check HIVE_HOME/conf
    if (ivysettingsPath == null || !(new File(ivysettingsPath).exists())) {
      if (System.getenv().containsKey(HIVE_HOME)) {
        ivysettingsPath = System.getenv().get(HIVE_HOME) + "/conf/ivysettings.xml";
      }
    }

    // If HIVE_HOME is not defined or file is not found in HIVE_HOME/conf then load default ivysettings.xml from class loader
    if (ivysettingsPath == null || !(new File(ivysettingsPath).exists())) {
      URL ivysetttingsResource = ClassLoader.getSystemResource("ivysettings.xml");
      if (ivysetttingsResource != null){
        ivysettingsPath = ivysetttingsResource.getFile();
        _console.printInfo("ivysettings.xml file not found in HIVE_HOME or HIVE_CONF_DIR," + ivysettingsPath + " will be used");
      }
    }

  }

  /**
   *
   * @param uri
   * @return List of URIs of downloaded jars
   * @throws URISyntaxException
   * @throws IOException
   */
  public List<URI> downloadDependencies(URI uri) throws URISyntaxException, IOException {
    Map<String, Object> dependencyMap = new HashMap<String, Object>();
    String authority = uri.getAuthority();
    if (authority == null) {
      throw new URISyntaxException(authority, "Invalid url: Expected 'org:module:version', found null");
    }
    String[] authorityTokens = authority.split(":");

    if (authorityTokens.length != 3) {
      throw new URISyntaxException(authority, "Invalid url: Expected 'org:module:version', found " + authority);
    }

    dependencyMap.put("org", authorityTokens[0]);
    dependencyMap.put("module", authorityTokens[1]);
    dependencyMap.put("version", authorityTokens[2]);
    Map<String, Object> queryMap = parseQueryString(uri.getQuery());
    if (queryMap != null) {
      dependencyMap.putAll(queryMap);
    }
    return grab(dependencyMap);
  }

  /**
   * @param queryString
   * @return queryMap Map which contains grape parameters such as transitive, exclude, ext and classifier.
   * Example: Input:  ext=jar&exclude=org.mortbay.jetty:jetty&transitive=true
   *          Output:  {[ext]:[jar], [exclude]:{[group]:[org.mortbay.jetty], [module]:[jetty]}, [transitive]:[true]}
   * @throws URISyntaxException
   */
  private Map<String, Object> parseQueryString(String queryString) throws URISyntaxException {
    if (queryString == null || queryString.isEmpty()) {
      return null;
    }
    List<Map<String, String>> excludeList = new LinkedList<Map<String, String>>();
    Map<String, Object> queryMap = new HashMap<String, Object>();
    String[] mapTokens = queryString.split("&");
    for (String tokens : mapTokens) {
      String[] mapPair = tokens.split("=");
      if (mapPair.length != 2) {
        throw new RuntimeException("Invalid query string: " + queryString);
      }
      if (mapPair[0].equals("exclude")) {
        excludeList.addAll(computeExcludeList(mapPair[1]));
      } else if (mapPair[0].equals("transitive")) {
        if (mapPair[1].toLowerCase().equals("true")) {
          queryMap.put(mapPair[0], true);
        } else {
          queryMap.put(mapPair[0], false);
        }
      } else {
        queryMap.put(mapPair[0], mapPair[1]);
      }
    }
    if (!excludeList.isEmpty()) {
      queryMap.put("exclude", excludeList);
    }
    return queryMap;
  }

  private List<Map<String, String>> computeExcludeList(String excludeString) throws URISyntaxException {
    String excludes[] = excludeString.split(",");
    List<Map<String, String>> excludeList = new LinkedList<Map<String, String>>();
    for (String exclude : excludes) {
      Map<String, String> tempMap = new HashMap<String, String>();
      String args[] = exclude.split(":");
      if (args.length != 2) {
        throw new URISyntaxException(excludeString,
            "Invalid exclude string: expected 'org:module,org:module,..', found " + excludeString);
      }
      tempMap.put("group", args[0]);
      tempMap.put("module", args[1]);
      excludeList.add(tempMap);
    }
    return excludeList;
  }

  /**
   *
   * @param dependencies
   * @return List of URIs of downloaded jars
   * @throws IOException
   */
  private List<URI> grab(Map<String, Object> dependencies) throws IOException {
    Map<String, Object> args = new HashMap<String, Object>();
    URI[] localUrls;

    //grape expects excludes key in args map
    if (dependencies.containsKey("exclude")) {
      args.put("excludes", dependencies.get("exclude"));
    }

    //Set transitive to true by default
    if (!dependencies.containsKey("transitive")) {
      dependencies.put("transitive", true);
    }

    args.put("classLoader", new GroovyClassLoader());
    System.setProperty("grape.config", ivysettingsPath);
    localUrls = Grape.resolve(args, dependencies);
    if (localUrls == null) {
      throw new IOException("Not able to download all the dependencies..");
    }
    return Arrays.asList(localUrls);
  }
}