Skip to content

Commit

Permalink
Identify devDependencies for Yarn
Browse files Browse the repository at this point in the history 
CLOUDBLD-3914

Use data from package.json to identify devDependencies in Yarn repos.

Signed-off-by: Daniel Cho <dacho@redhat.com>
  • Loading branch information
@dcho5 @brunoapimentel
dcho5 authored and brunoapimentel committed Dec 15, 2021
1 parent d429dc6 commit 2d19dce
Show file tree
Hide file tree
Showing 5 changed files with 269 additions and 78 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,7 @@ Feature | gomod | npm | pip | yarn |
Baseline | ✓ | ✓ | ✓ | ✓ |
Content Manifest | ✓ | ✓ | ✓ | ✓ |
Dependency Replacements | ✓ | x | x | x |
Dev Dependencies | ✓ | ✓ | ✓ | x |
Dev Dependencies | ✓ | ✓ | ✓ | ✓ |
External Dependencies | N/A | ✓ | ✓ | ✓ |
Multiple Paths | ✓ | ✓ | ✓ | ✓ |
Nested Dependencies | ✓ | ✓ | x | ✓ |
Expand Down
73 changes: 62 additions & 11 deletions cachito/workers/pkg_managers/yarn.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import copy
import json
import logging
from collections import deque
from os.path import normpath
from pathlib import Path
from typing import Dict, Optional
Expand Down Expand Up @@ -69,7 +70,41 @@ def get_yarn_proxy_repo_username(request_id):
return f"cachito-yarn-{request_id}"


def _get_deps(yarn_lock, file_deps_allowlist):
def _find_reachable_deps(visited_deps, dep, yarn_lock):
"""
Get a set of all dependencies reachable by a top-level dependency using BFS.
:param yarn_lock: yarn.lock file as a dictionary
:param dep: top-level dependency in package.json
:param reachable_deps: set of already visited non-dev dependencies
"""
yarn_lock_parsed = _expand_yarn_lock_keys(yarn_lock)
visited_deps.add(dep)
bfs_queue = deque([dep])
while bfs_queue:
current_dep = bfs_queue.popleft()
package = pyarn.lockfile.Package.from_dict(current_dep, yarn_lock_parsed[current_dep])
bfs_queue.extend(
f"{name}@{version}"
for name, version in package.dependencies.items()
if f"{name}@{version}" not in visited_deps
)
visited_deps.add(current_dep)


def _split_yarn_lock_key(dep_identifer):
"""
Remove unnecessary quotes in dep_identifier and split the string into a list of dependencies.
String dep_identifer contains one or more dependencies separated by commas.
:param dep_identifier: a string which lists all of the dependencies in the identifer
:return: a list of all the dependencies in the identifier
"""
return dep_identifer.replace('"', "").split(", ")


def _get_deps(package_json, yarn_lock, file_deps_allowlist):
"""
Process the dependencies in a yarn.lock file and return relevant information.
Expand All @@ -90,10 +125,25 @@ def _get_deps(yarn_lock, file_deps_allowlist):
"""
deps = []
nexus_replacements = {}
non_dev_deps = set()

for dep_type in ["dependencies", "peerDependencies", "optionalDependencies"]:
if dep_type not in package_json:
continue
for name, version in package_json[dep_type].items():
dep = f"{name}@{version}"
if dep not in non_dev_deps:
_find_reachable_deps(non_dev_deps, dep, yarn_lock)

for dep_identifier, dep_data in yarn_lock.items():
package = pyarn.lockfile.Package.from_dict(dep_identifier, dep_data)

dev = True
for dep_id in _split_yarn_lock_key(dep_identifier):
if dep_id in non_dev_deps:
dev = False
break

if package.url:
source = package.url
elif package.relpath:
Expand All @@ -116,7 +166,7 @@ def _get_deps(yarn_lock, file_deps_allowlist):

dep = {
"bundled": False, # yarn.lock does not seem to contain bundled deps at all
# "dev": <yarn.lock does not state whether a dependency is dev>
"dev": dev,
"name": package.name,
"version_in_nexus": nexus_replacement["version"] if nexus_replacement else None,
"type": "yarn",
Expand Down Expand Up @@ -234,7 +284,7 @@ def _get_package_and_deps(package_json_path, yarn_lock_path):
get_worker_config().cachito_yarn_file_deps_allowlist.get(package["name"], [])
)

deps, nexus_replacements = _get_deps(yarn_lock, file_deps_allowlist)
deps, nexus_replacements = _get_deps(package_json, yarn_lock, file_deps_allowlist)
return {
"package": package,
"deps": deps,
Expand Down Expand Up @@ -284,9 +334,9 @@ def _set_proxy_resolved_urls(yarn_lock: Dict[str, dict], proxy_repo_name: str) -
return modified


def _expand_replacements(nexus_replacements: Dict[str, dict]) -> Dict[str, dict]:
def _expand_yarn_lock_keys(nexus_replacements: Dict[str, dict]) -> Dict[str, dict]:
"""
Expand all N:1 keys in the Nexus replacements dict into N 1:1 keys.
Expand all N:1 keys in the yarn.lock dict into N 1:1 keys.
In the original dict, 1 key may in fact be N comma-separated keys. These N keys all have the
same value, making them N:1 keys. In the expanded dict, these will be turned into N 1:1 keys.
Expand All @@ -297,12 +347,12 @@ def _expand_replacements(nexus_replacements: Dict[str, dict]) -> Dict[str, dict]
:param dict nexus_replacements: a dict of nexus replacements which may contain N:1 keys
:return: a dict of nexus replacements where all N:1 keys have been expanded to N 1:1 keys
"""
expanded_replacements = {
expanded_yarn_lock_keys = {
key: nexus_replacements[multi_key]
for multi_key in nexus_replacements
for key in map(str.strip, multi_key.split(","))
for key in _split_yarn_lock_key(multi_key)
}
return expanded_replacements
return expanded_yarn_lock_keys


def _match_to_new_version(
Expand All @@ -313,7 +363,8 @@ def _match_to_new_version(
:param str dep_name: dependency name
:param str dep_version: dependency version
:param dict expanded_replacements: expanded dict of Nexus replacements, see _expand_replacements
:param dict expanded_replacements: expanded dict of Nexus replacements,
see _expand_yarn_lock_keys
:return: new version (str) or None
"""
dep_identifier = f"{dep_name}@{dep_version}"
Expand All @@ -329,7 +380,7 @@ def _replace_deps_in_package_json(package_json, nexus_replacements):
{<dependency identifier>: <dependency info>}
:return: copy of package.json data with replacements applied (or None if no replacements match)
"""
expanded_replacements = _expand_replacements(nexus_replacements)
expanded_replacements = _expand_yarn_lock_keys(nexus_replacements)

package_json_new = copy.deepcopy(package_json)
modified = False
Expand Down Expand Up @@ -367,7 +418,7 @@ def _replace_deps_in_yarn_lock(yarn_lock, nexus_replacements):
{<dependency identifier>: <dependency info>}
:return: copy of yarn.lock data with replacements applied
"""
expanded_replacements = _expand_replacements(nexus_replacements)
expanded_replacements = _expand_yarn_lock_keys(nexus_replacements)
yarn_lock_new = {}

for key, value in yarn_lock.items():
Expand Down
60 changes: 40 additions & 20 deletions tests/integration/test_data/cached_dependencies.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -761,85 +761,105 @@ yarn_cached_deps:
# Parts of the Cachito response to check
response_expectations:
dependencies:
- name: assertion-error
- dev: false
name: assertion-error
replaces: null
type: yarn
version: 1.1.0
- name: chai
- dev: false
name: chai
replaces: null
type: yarn
version: 4.2.0
- name: check-error
- dev: false
name: check-error
replaces: null
type: yarn
version: 1.0.2
- name: deep-eql
- dev: false
name: deep-eql
replaces: null
type: yarn
version: 3.0.1
- name: fecha
- dev: false
name: fecha
replaces: null
type: yarn
version: https://github.com/taylorhakes/fecha/archive/91680e4db1415fea33eac878cfd889c80a7b55c7.tar.gz#f09ea0b8115b9733dddc88227086c73ba4ddc926
- name: get-func-name
- dev: false
name: get-func-name
replaces: null
type: yarn
version: 2.0.0
- name: npm-test-project
- dev: false
name: npm-test-project
replaces: null
type: yarn
version: git+https://gitlab.com/eakhmano-test-group/eakhmano-test-subgroup/subsubgroup/npm-test-project.git#88114c48d7d894f7e2306adf06137b096e9cdf29
- name: pathval
- dev: false
name: pathval
replaces: null
type: yarn
version: 1.1.1
- name: test-npm-dep
- dev: false
name: test-npm-dep
replaces: null
type: yarn
version: git+https://elinah@bitbucket.org/elinah/test-npm-dep.git#b952715c0706f30d92a7253dcdadc510a618eca5
- name: type-detect
- dev: false
name: type-detect
replaces: null
type: yarn
version: 4.0.8
packages:
- dependencies:
- name: assertion-error
- dev: false
name: assertion-error
replaces: null
type: yarn
version: 1.1.0
- name: chai
- dev: false
name: chai
replaces: null
type: yarn
version: 4.2.0
- name: check-error
- dev: false
name: check-error
replaces: null
type: yarn
version: 1.0.2
- name: deep-eql
- dev: false
name: deep-eql
replaces: null
type: yarn
version: 3.0.1
- name: fecha
- dev: false
name: fecha
replaces: null
type: yarn
version: https://github.com/taylorhakes/fecha/archive/91680e4db1415fea33eac878cfd889c80a7b55c7.tar.gz#f09ea0b8115b9733dddc88227086c73ba4ddc926
- name: get-func-name
- dev: false
name: get-func-name
replaces: null
type: yarn
version: 2.0.0
- name: npm-test-project
- dev: false
name: npm-test-project
replaces: null
type: yarn
version: git+https://gitlab.com/eakhmano-test-group/eakhmano-test-subgroup/subsubgroup/npm-test-project.git#88114c48d7d894f7e2306adf06137b096e9cdf29
- name: pathval
- dev: false
name: pathval
replaces: null
type: yarn
version: 1.1.1
- name: test-npm-dep
- dev: false
name: test-npm-dep
replaces: null
type: yarn
version: git+https://elinah@bitbucket.org/elinah/test-npm-dep.git#b952715c0706f30d92a7253dcdadc510a618eca5
- name: type-detect
- dev: false
name: type-detect
replaces: null
type: yarn
version: 4.0.8
Expand Down
Loading

0 comments on commit 2d19dce

Please sign in to comment.