Git URLs support in npm-shrinkwrap

Change-Id: I602581a38b0b0006ca3e9dadcec397057e0bb510

lib/vcap/staging/plugin/git_cache.rb

@@ -16,7 +16,8 @@ def get_source(source, dst_dir)
     return unless source[:uri] && source[:revision]
     uri = normalize_uri(source[:uri])
     revision = source[:revision].strip
-    return unless revision =~ /^[a-z0-9]+$/
+    # Check that revision follows git-check-ref-format
+    return unless revision =~ /^[a-z0-9_\-\/\.]*[a-z0-9_\-]$/
 
     cached_path = find_source(uri, revision)
     return unless cached_path && File.directory?(cached_path)

lib/vcap/staging/plugin/node/npm_support/npm_package.rb

@@ -1,39 +1,82 @@
 require "fileutils"
+require "uri"
+require File.expand_path("../../../secure_operations", __FILE__)
 
+# Node module class
+# Describes node module and performs operations: build, fetch and install
 class NpmPackage
+  include SecureOperations
 
-  def initialize(name, version, where, secure_uid, secure_gid,
-                 npm_helper, logger, cache)
+  def initialize(name, props, where, secure_uid, secure_gid,
+                 npm_helper, logger, cache, git_cache)
     @name  = name.chomp
-    @version = version.chomp
+    @target = (props["from"] || props["version"]).chomp
     @npm_helper = npm_helper
-    @secure_uid = secure_uid
-    @secure_gid = secure_gid
+    @uid = secure_uid
+    @gid = secure_gid
     @logger = logger
     @cache = cache
+    @git_cache = git_cache
     @dst_dir = File.join(where, "node_modules", @name)
   end
 
   def install
+    # Parsing source target according to npm source
+    # (https://github.com/isaacs/npm/blob/master/lib/install.js#resolver):
+    # If there is a "from" field use it as target
+    # Else use "version" as target
+    # If "version" starts with "http" or "git" it's a git URL
+    # Else it is fetched from npm registry
     if url_provided?
-      @logger.warn("Failed installing package #{@name}. URLs are not supported")
+      install_from_git
+    else
+      install_from_registry
+    end
+  end
+
+  def install_from_git
+    # We need to parse URL to get git repo URL and reference (commit SHA, tag, branch)
+    begin
+      parsed_url = URI(@target)
+    rescue => e
+      @logger.warn("Error parsing module source URL: #{e.message}")
       return nil
     end
+    ref = parsed_url.fragment || "master"
+    git_url = @target.sub(/#.*$/, "")
+
+    fetched = fetch_from_git(git_url, ref)
+    unless fetched
+      @logger.warn("Failed fetching module #{@name}@#{@target} from Git source")
+      return nil
+    end
+
+    # Unlike gemspec package.json does not provide information if module has native extensions
+    # So we build everything
+    installed = build(fetched)
+    if installed
+      return @dst_dir if copy_to_dst(installed)
+    end
+  end
 
-    cached = @cache.get(@name, @version)
+  def install_from_registry
+    cached = @cache.get(@name, @target)
     if cached
       return @dst_dir if copy_to_dst(cached)
 
     else
       @registry_data = get_registry_data
 
       unless @registry_data.is_a?(Hash) && @registry_data["version"]
-        log_name = @version.empty? ? @name : "#{@name}@#{@version}"
+        log_name = @target.empty? ? @name : "#{@name}@#{@target}"
         @logger.warn("Failed getting the requested package: #{log_name}")
         return nil
       end
 
-      installed = fetch_build
+      fetched = fetch_from_registry(@registry_data["source"])
+      return unless fetched
+
+      installed = build(fetched)
 
       if installed
         cached = @cache.put(installed, @name, @registry_data["version"])
@@ -42,22 +85,29 @@ def install
     end
   end
 
-  def fetch(source, where)
+  def fetch_from_git(uri, ref)
+    tmp_dir = mk_temp_dir
+    source = {}
+    source[:uri] = uri
+    source[:revision] = ref
+    @git_cache.get_source(source, tmp_dir)
+  end
+
+  def fetch_from_registry(source)
+    where = mk_temp_dir
     Dir.chdir(where) do
       fetched_tarball = "package.tgz"
       cmd = "wget --quiet --retry-connrefused --connect-timeout=5 " +
         "--no-check-certificate --output-document=#{fetched_tarball} #{source}"
       `#{cmd}`
       return unless $?.exitstatus == 0
 
-      package_dir = File.join(where, "package")
-      FileUtils.mkdir_p(package_dir)
-
       fetched_path = File.join(where, fetched_tarball)
-      `tar xzf #{fetched_path} --directory=#{package_dir} --strip-components=1 2>&1`
+      `tar xzf #{fetched_path} --directory=#{where} --strip-components=1 2>&1`
       return unless $?.exitstatus == 0
+      FileUtils.rm_rf(fetched_path)
 
-      File.exists?(package_dir) ? package_dir : nil
+      File.exists?(where) ? where : nil
     end
   end
 
@@ -69,67 +119,25 @@ def copy_to_dst(source)
     $?.exitstatus == 0
   end
 
-  # This is done in a similar to ruby gems way until PackageCache is available
-
-  def fetch_build
-    tmp_dir = Dir.mktmpdir
-    at_exit do
-      user = `whoami`.chomp
-      `sudo /bin/chown -R #{user} #{tmp_dir}` if @secure_uid
-      FileUtils.rm_rf(tmp_dir)
-    end
-
-    package_dir = fetch(@registry_data["source"], tmp_dir)
-    return unless package_dir
-
-    if @secure_uid
-      chown_cmd = "sudo /bin/chown -R #{@secure_uid}:#{@secure_gid} #{tmp_dir} 2>&1"
-      chown_output = `#{chown_cmd}`
-
-      if $?.exitstatus != 0
-        @logger.error("Failed chowning install dir: #{chown_output}")
-        return nil
-      end
-    end
-
+  def build(package_dir)
     cmd = @npm_helper.build_cmd(package_dir)
+    cmd_status, output = run_secure(cmd, package_dir, :secure_group => true)
 
-    if @secure_uid
-      cmd ="sudo -u '##{@secure_uid}' sg #{secure_group} -c \"cd #{tmp_dir} && #{cmd}\" 2>&1"
-    else
-      cmd ="cd #{tmp_dir} && #{cmd}"
-    end
-
-    output = nil
-    IO.popen(cmd) do |io|
-      output = io.read
-    end
-    child_status = $?.exitstatus
-
-    if child_status != 0
+    if cmd_status != 0
       @logger.warn("Failed installing package: #{@name}")
       if output =~ /npm not ok/
         output.lines.grep(/^npm ERR! message/) do |error_message|
           @logger.warn(error_message.chomp)
         end
       end
     end
-
-    if @secure_uid
-      # Kill any stray processes that the npm compilation may have created
-      `sudo -u '##{@secure_uid}' pkill -9 -U #{@secure_uid} 2>&1`
-      me = `whoami`.chomp
-      `sudo chown -R #{me} #{tmp_dir}`
-      @logger.debug("Failed chowning #{tmp_dir} to #{me}") if $?.exitstatus != 0
-    end
-
-    return package_dir if child_status == 0
+    cmd_status == 0 ? package_dir : nil
   end
 
   def get_registry_data
     # TODO: 1. make direct request, we need only tarball source
     # 2. replicate npm registry database
-    package_link = "#{@name}@\"#{@version}\""
+    package_link = "#{@name}@\"#{@target}\""
     output = `#{@npm_helper.versioner_cmd(package_link)} 2>&1`
     if $?.exitstatus != 0 || output.empty?
       return nil
@@ -140,17 +148,20 @@ def get_registry_data
         return nil
       end
     end
-    return resolved
+    resolved
   end
 
   private
 
   def url_provided?
-    @version =~ /^http/ or @version =~ /^git/
+    @target =~ /^http/ or @target =~ /^git/
   end
 
-  def secure_group
-    group_name = `awk -F: '{ if ( $3 == #{@secure_gid} ) { print $1 } }' /etc/group`
-    group_name.chomp
+  def mk_temp_dir
+    tmp_dir = Dir.mktmpdir
+    at_exit do
+      secure_delete(tmp_dir)
+    end
+    tmp_dir
   end
 end

lib/vcap/staging/plugin/node/npm_support/npm_support.rb

@@ -4,6 +4,7 @@
 require File.expand_path("../npm_cache", __FILE__)
 require File.expand_path("../npm_package", __FILE__)
 require File.expand_path("../npm_helper", __FILE__)
+require File.expand_path("../../../git_cache", __FILE__)
 
 module NpmSupport
 
@@ -29,9 +30,11 @@ def compile_node_modules
 
     cache_base_dir = StagingPlugin.platform_config["cache"]
     FileUtils.mkdir_p File.join(cache_base_dir, "node_modules")
-    cache_dir  = File.join(cache_base_dir, "node_modules", library_version)
-    @cache = NpmCache.new(cache_dir, logger)
+    cache_version_dir  = File.join(cache_base_dir, "node_modules", library_version)
+    @cache = NpmCache.new(cache_version_dir, logger)
 
+    @git_cache = GitCache.new(File.join(cache_base_dir, "git_cache"),
+                              File.join(cache_version_dir, "git_cache"), logger)
     logger.info("Installing dependencies. Node version #{runtime[:version]}")
     install_packages(@dependencies, app_directory)
   end
@@ -47,8 +50,8 @@ def should_install_packages?
 
   def install_packages(dependencies, where)
     dependencies.each do |name, props|
-      package = NpmPackage.new(name, props["version"], where, @staging_uid,
-                               @staging_gid, @npm_helper, logger, @cache)
+      package = NpmPackage.new(name, props, where, @staging_uid,
+                               @staging_gid, @npm_helper, logger, @cache, @git_cache)
       installed_dir = package.install
       if installed_dir && props["dependencies"].is_a?(Hash)
         install_packages(props["dependencies"], installed_dir)

lib/vcap/staging/plugin/secure_operations.rb

@@ -7,14 +7,18 @@ module SecureOperations
   # Run a process as a secure user, if @uid is set.
   # Otherwise, process is run as current user
   # Use the "where" variable to set the process working dir,
-  def run_secure(cmd, where)
+  def run_secure(cmd, where, options={})
     exitstatus = nil
     output = nil
 
     secure_file(where)
     begin
       if @uid
-        cmd = "cd #{where} && sudo -u '##{@uid}' #{cmd}"
+        if options[:secure_group]
+          cmd ="sudo -u '##{@uid}' sg #{secure_group} -c \"cd #{where} && #{cmd}\" 2>&1"
+        else
+          cmd = "cd #{where} && sudo -u '##{@uid}' #{cmd}"
+        end
       else
         cmd = "cd #{where} && #{cmd}"
       end
@@ -46,7 +50,8 @@ def secure_file(file)
       if $?.exitstatus != 0
         raise "Failed chmodding dir: #{chmod_output}"
       end
-      chown_output = `sudo /bin/chown -R #{@uid} #{file} 2>&1`
+      chown_user = @gid ? "#{@uid}:#{@gid}" : @uid
+      chown_output = `sudo /bin/chown -R #{chown_user} #{file} 2>&1`
       if $?.exitstatus != 0
         raise "Failed chowning dir: #{chown_output}"
       end
@@ -57,8 +62,12 @@ def secure_file(file)
   # to current user
   def unsecure_file(file)
     if @uid
-      user = `whoami`.chomp
-      chown_output = `sudo /bin/chown -R #{user} #{file} 2>&1`
+      chown_user = `id -u`.chomp
+      if @gid
+        user_group = `id -g`.chomp
+        chown_user = "#{chown_user}:#{user_group}"
+      end
+      chown_output = `sudo /bin/chown -R #{chown_user} #{file} 2>&1`
       if $?.exitstatus != 0
         raise "Failed chowning dir: #{chown_output}"
       end
@@ -68,8 +77,14 @@ def unsecure_file(file)
   # Change ownership of file back to current user
   # and delete file
   def secure_delete(file)
-    user = `whoami`.chomp
-    `sudo /bin/chown -R #{user} #{file}` if @uid
-    FileUtils.rm_rf(file)
+    if File.exists?(file)
+      unsecure_file(file)
+      FileUtils.rm_rf(file)
+    end
+  end
+
+  def secure_group
+    group_name = `awk -F: '{ if ( $3 == #{@gid} ) { print $1 } }' /etc/group`
+    group_name.chomp
   end
 end

spec/fixtures/apps/node_deps_git/source/app.js

@@ -0,0 +1,8 @@
+require("graceful-fs");
+
+var port = process.env.VCAP_APP_PORT || 3000;
+
+require("http").createServer(function (req, res) {
+  res.writeHead(200, {"Content-Type" : "text/html"});
+  res.end("Hello from Cloud!");
+}).listen(port);

spec/fixtures/apps/node_deps_git/source/package.json

@@ -0,0 +1,7 @@
+{
+  "name" : "node-git",
+  "version" : "0.1.0",
+  "dependencies" : {
+    "graceful-fs" : "*"
+  }
+}

spec/support/staging_spec_helpers.rb

@@ -56,7 +56,6 @@ def stage(env = {})
                    app_source
                  end
     env[:environment] ||= []
-
     runtime_name = env[:runtime_info][:name].upcase
     if ENV["VCAP_RUNTIME_#{runtime_name}"]
       env[:runtime_info][:executable] = ENV["VCAP_RUNTIME_#{runtime_name}"]
@@ -77,5 +76,4 @@ def stage(env = {})
     FileUtils.rm_r(working_dir) if working_dir
     FileUtils.rm_r(source_tempdir) if source_tempdir
   end
-end
-
+end