Merge pull request #232 from github/yarn-source

Add yarn source

.github/workflows/test.yml

@@ -342,3 +342,35 @@ jobs:
       run: script/source-setup/pipenv
     - name: Run tests
       run: script/test pipenv
+
+  yarn:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # not using 1.0.0 because it doesn't support `yarn list --production`
+        yarn_version: [ 1.4.0, latest ]
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup node
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12
+    - name: Install Yarn
+      run: npm install -g yarn@${YARN_VERSION}
+      env:
+        YARN_VERSION: ${{ matrix.yarn_version }}
+    - name: Set up Ruby
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - run: bundle lock
+    - uses: actions/cache@preview
+      with:
+        path: vendor/gems
+        key: ${{ runner.os }}-gem-2.6.x-${{ hashFiles(format('{0}{1}', github.workspace, '/Gemfile.lock')) }}
+    - name: Bootstrap
+      run: script/bootstrap
+    - name: Set up fixtures
+      run: script/source-setup/yarn
+    - name: Run tests
+      run: script/test yarn

.gitignore

@@ -12,26 +12,39 @@
 test/fixtures/bundler/.bundle/
 test/fixtures/bundler/vendor/
 test/fixtures/bundler/Gemfile.lock
+
 test/fixtures/bower/bower_components
+
 test/fixtures/npm/node_modules
 test/fixtures/npm/package-lock.json
+
 test/fixtures/go/src/*
 test/fixtures/go/pkg
 !test/fixtures/go/src/test
 !test/fixtures/go/src/modules_test
+
 test/fixtures/cabal/*
 !test/fixtures/cabal/app*
+
 test/fixtures/git_submodule/*
 !test/fixtures/git_submodule/README
+
 test/fixtures/pip/venv
+
 test/fixtures/pipenv/Pipfile.lock
+
 !test/fixtures/migrations/**/*
+
 test/fixtures/composer/**/*
 !test/fixtures/composer/composer.json
+
 test/fixtures/mix/_build
 test/fixtures/mix/deps
 test/fixtures/mix/mix.lock
 
+test/fixtures/yarn/*
+!test/fixtures/yarn/package.json
+
 vendor/licenses
 .licenses
 *.gem

README.md

@@ -25,11 +25,11 @@ See the [migration documentation](./docs/migrating_to_newer_versions.md) for mor
 
 Licensed uses the `libgit2` bindings for Ruby provided by `rugged`. `rugged` requires `cmake` and `pkg-config` which you may need to install before you can install Licensed.
 
-   >  Ubuntu 
+   >  Ubuntu
     
     sudo apt-get install cmake pkg-config
 
-   >  OS X 
+   >  OS X
        
     brew install cmake pkg-config
 
@@ -110,6 +110,7 @@ Dependencies will be automatically detected for all of the following sources by
 1. [Pipenv](./docs/sources/pipenv.md)
 1. [Git Submodules (git_submodule)](./docs/sources/git_submodule.md)
 1. [Mix](./docs/sources/mix.md)
+1. [Yarn](./docs/sources/yarn.md)
 
 You can disable any of them in the configuration file:
 

docs/sources/yarn.md

@@ -0,0 +1,16 @@
+# Yarn
+
+The yarn source will detect dependencies when `package.json` and `yarn.lock` are found at an app's `source_path`.
+
+It uses `yarn list` to enumerate dependencies and `yarn info` to get metadata on each package.
+
+### Including development dependencies
+
+Yarn versions < 1.3.0 will always include non-production dependencies due to a bug in those yarn versions.
+
+Starting with yarn version >= 1.3.0, the yarn source excludes non-production dependencies by default.  To include development and test dependencies, set `production_only: false` in `.licensed.yml`.
+
+```yml
+yarn:
+  production_only: false
+```

lib/licensed/sources.rb

@@ -15,5 +15,6 @@ module Sources
     require "licensed/sources/pipenv"
     require "licensed/sources/gradle"
     require "licensed/sources/mix"
+    require "licensed/sources/yarn"
   end
 end

lib/licensed/sources/yarn.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+require "json"
+
+module Licensed
+  module Sources
+    class Yarn < Source
+      def enabled?
+        return unless Licensed::Shell.tool_available?("yarn") && Licensed::Shell.tool_available?("npm")
+
+        config.pwd.join("package.json").exist? && config.pwd.join("yarn.lock").exist?
+      end
+
+      def enumerate_dependencies
+        packages.map do |name, package|
+          Dependency.new(
+            name: name,
+            version: package["version"],
+            path: package["path"],
+            metadata: {
+              "type"     => Yarn.type,
+              "name"     => package["name"],
+              "summary"  => package["description"],
+              "homepage" => package["homepage"]
+            }
+          )
+        end
+      end
+
+      def packages
+        root_dependencies = JSON.parse(yarn_list_command)["data"]["trees"]
+        root_path = config.pwd
+        all_dependencies = {}
+        recursive_dependencies(root_path, root_dependencies).each do |name, results|
+          results.uniq! { |package| package["version"] }
+          if results.size == 1
+            all_dependencies[name] = results[0]
+          else
+            results.each do |package|
+              all_dependencies[package["id"].sub("@", "-")] = package
+            end
+          end
+        end
+
+        Parallel.map(all_dependencies) { |name, dep| [name, package_info(dep)] }.to_h
+      end
+
+      # Recursively parse dependency JSON data.  Returns a hash mapping the
+      # package name to it's metadata
+      def recursive_dependencies(path, dependencies, result = {})
+        dependencies.each do |dependency|
+          next if dependency["shadow"]
+          name, version = dependency["name"].split("@")
+
+          dependency_path = path.join("node_modules", name)
+          (result[name] ||= []) << {
+            "id" => dependency["name"],
+            "name" => name,
+            "version" => version,
+            "path" => dependency_path
+          }
+          recursive_dependencies(dependency_path, dependency["children"], result)
+        end
+        result
+      end
+
+      # Returns the output from running `yarn list` to get project dependencies
+      def yarn_list_command
+        args = %w(--json -s --no-progress)
+        args << "--production" unless include_non_production?
+        Licensed::Shell.execute("yarn", "list", *args, allow_failure: true)
+      end
+
+      # Returns extended information for the package
+      def package_info(package)
+        info = package_info_command(package["id"])
+        return package if info.nil? || info.empty?
+
+        info = JSON.parse(info)["data"]
+        package.merge(
+          "description" => info["description"],
+          "homepage" => info["homepage"]
+        )
+      end
+
+      # Returns the output from running `yarn info` to get package info
+      def package_info_command(id)
+        Licensed::Shell.execute("yarn", "info", "-s", "--json", id, allow_failure: true)
+      end
+
+      # Returns whether to include non production dependencies based on the licensed configuration settings
+      def include_non_production?
+        config.dig("yarn", "production_only") == false
+      end
+    end
+  end
+end

script/source-setup/yarn

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [ -z "$(which yarn)" ]; then
+  echo "A local yarn installation is required for yarn development." >&2
+  exit 127
+fi
+
+# setup test fixtures
+BASE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd $BASE_PATH/test/fixtures/yarn
+
+if [ "$1" == "-f" ]; then
+  find . -not -regex "\.*" -and -not -name "package\.json" -print0 | xargs -0 rm -rf
+fi
+
+yarn install

test/fixtures/command/yarn.yml

@@ -0,0 +1,3 @@
+expected_dependency: autoprefixer
+source_path: test/fixtures/yarn
+cache_path: test/fixtures/yarn/.licenses

test/fixtures/yarn/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "fixtures",
+  "version": "1.0.0",
+  "dependencies": {
+    "autoprefixer": "5.2.0"
+  },
+  "devDependencies": {
+    "string.prototype.startswith": "0.2.0"
+  },
+  "description": "npm test fixture",
+  "repository": "https://github.com/github/licensed",
+  "license": "MIT"
+}

test/sources/yarn_test.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+require "test_helper"
+require "tmpdir"
+require "fileutils"
+
+if Licensed::Shell.tool_available?("yarn")
+  describe Licensed::Sources::Yarn do
+    let(:config) { Licensed::Configuration.new }
+    let(:fixtures) { File.expand_path("../../fixtures/yarn", __FILE__) }
+    let(:source) { Licensed::Sources::Yarn.new(config) }
+
+    describe "enabled?" do
+      it "is true if package.json and yarn.lock exists" do
+        Dir.mktmpdir do |dir|
+          Dir.chdir(dir) do
+            File.write "package.json", ""
+            File.write "yarn.lock", ""
+            assert source.enabled?
+          end
+        end
+      end
+
+      it "is false if package.json does not exist" do
+        Dir.mktmpdir do |dir|
+          Dir.chdir(dir) do
+            File.write "yarn.lock", ""
+            refute source.enabled?
+          end
+        end
+      end
+
+      it "is false if yarn.lock does not exist" do
+        Dir.mktmpdir do |dir|
+          Dir.chdir(dir) do
+            File.write "package.json", ""
+            refute source.enabled?
+          end
+        end
+      end
+    end
+
+    describe "dependencies" do
+      it "includes declared dependencies" do
+        Dir.chdir fixtures do
+          dep = source.dependencies.detect { |d| d.name == "autoprefixer" }
+          assert dep
+          assert_equal "yarn", dep.record["type"]
+          assert_equal "5.2.0", dep.version
+          assert dep.record["homepage"]
+          assert dep.record["summary"]
+        end
+      end
+
+      it "includes indirect dependencies" do
+        Dir.chdir fixtures do
+          assert source.dependencies.detect { |dep| dep.name == "autoprefixer-core" }
+        end
+      end
+
+      it "does not include dev dependencies by default" do
+        Dir.chdir fixtures do
+          refute source.dependencies.detect { |dep| dep.name == "string.prototype.startswith" }
+        end
+      end
+
+      it "includes dev dependencies if configured" do
+        Dir.chdir fixtures do
+          config["yarn"] = { "production_only" => false }
+          assert source.dependencies.detect { |dep| dep.name == "string.prototype.startswith" }
+        end
+      end
+
+      it "does not include ignored dependencies" do
+        Dir.chdir fixtures do
+          config.ignore({ "type" => Licensed::Sources::Yarn.type, "name" => "autoprefixer" })
+          refute source.dependencies.detect { |dep| dep.name == "autoprefixer" }
+        end
+      end
+
+      describe "with multiple instances of a dependency" do
+        it "includes version in the dependency name for multiple unique versions" do
+          Dir.chdir fixtures do
+            graceful_fs_dependencies = source.dependencies.select { |dep| dep.name == "graceful-fs" }
+            assert_empty graceful_fs_dependencies
+
+            graceful_fs_dependencies = source.dependencies.select { |dep| dep.name =~ /graceful-fs/ }
+            assert_equal 2, graceful_fs_dependencies.size
+            graceful_fs_dependencies.each do |dependency|
+              assert_equal "#{dependency.record["name"]}-#{dependency.version}", dependency.name
+              assert dependency.exist?
+            end
+          end
+        end
+
+        it "does not include version in the dependency name for a single unique version" do
+          Dir.chdir fixtures do
+            dep = source.dependencies.detect { |d| d.name == "wrappy" }
+            assert dep
+            assert_equal "wrappy", dep.name
+          end
+        end
+      end
+    end
+  end
+end