Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: backport npm fetcher code #8

Open
wants to merge 3 commits into
base: dunfell
Choose a base branch
from
Open

fix: backport npm fetcher code #8

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2ff01e7
fix: backport npm fetcher code
Nov 17, 2021
a2a921d
fix: prevent failing to fetch from npm when trying to use the cache
Nov 17, 2021
27cde94
Merge branch 'YoeDistro:dunfell' into dunfell
Feb 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 48 additions & 24 deletions bitbake/lib/bb/fetch2/npm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,15 @@
#
"""
BitBake 'Fetch' npm implementation

npm fetcher support the SRC_URI with format of:
SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..."

Supported SRC_URI options are:

- package
The npm package name. This is a mandatory parameter.

- version
The npm package version. This is a mandatory parameter.

- downloadfilename
Specifies the filename used when storing the downloaded file.

- destsuffix
Specifies the directory to use to unpack the package (default: npm).
"""
Expand All @@ -40,6 +34,7 @@
from bb.fetch2 import runfetchcmd
from bb.utils import is_semver


def npm_package(package):
"""Convert the npm package name to remove unsupported character"""
# Scoped package names (with the @) use the same naming convention
Expand All @@ -48,14 +43,17 @@ def npm_package(package):
return re.sub("/", "-", package[1:])
return package


def npm_filename(package, version):
"""Get the filename of a npm package"""
return npm_package(package) + "-" + version + ".tgz"


def npm_localfile(package, version):
"""Get the local filename of a npm package"""
return os.path.join("npm2", npm_filename(package, version))


def npm_integrity(integrity):
"""
Get the checksum name and expected value from the subresource integrity
Expand All @@ -64,53 +62,79 @@ def npm_integrity(integrity):
algo, value = integrity.split("-", maxsplit=1)
return "%ssum" % algo, base64.b64decode(value).hex()


def npm_unpack(tarball, destdir, d):
"""Unpack a npm tarball"""
bb.utils.mkdirhier(destdir)
cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
cmd += " --no-same-owner"
cmd += " --delay-directory-restore"
cmd += " --strip-components=1"
runfetchcmd(cmd, d, workdir=destdir)
runfetchcmd("chmod -R +X %s" % (destdir), d, quiet=True, workdir=destdir)


class NpmEnvironment(object):
"""
Using a npm config file seems more reliable than using cli arguments.
This class allows to create a controlled environment for npm commands.
"""
def __init__(self, d, configs=None):

def __init__(self, d, configs=None, npmrc=None):
self.d = d
self.configs = configs

if configs:
self.user_config = tempfile.NamedTemporaryFile(
mode="w", buffering=1)
self.user_config_name = self.user_config.name

for key, value in configs:
self.user_config.write("%s=%s\n" % (key, value))
else:
self.user_config_name = ""

if npmrc:
self.global_config_name = npmrc
else:
self.global_config_name = "/dev/null"

def __del__(self):
if hasattr(self, "user_config"):
self.user_config.close()

def run(self, cmd, args=None, configs=None, workdir=None):
"""Run npm command in a controlled environment"""
with tempfile.TemporaryDirectory() as tmpdir:
d = bb.data.createCopy(self.d)
d.setVar("HOME", tmpdir)

cfgfile = os.path.join(tmpdir, "npmrc")

if not workdir:
d.setVar("HOME", tmpdir)
workdir = tmpdir
else:
d.setVar("HOME", workdir)

def _run(cmd):
cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd
cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd
if self.user_config_name != "":
cmd = "NPM_CONFIG_USERCONFIG=%s " % (
self.user_config_name) + cmd
else:
cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (
self.global_config_name) + cmd
return runfetchcmd(cmd, d, workdir=workdir)

if self.configs:
for key, value in self.configs:
_run("npm config set %s %s" % (key, shlex.quote(value)))

if configs:
bb.warn("Use of configs argument of NpmEnvironment.run() function"
" is deprecated. Please use args argument instead.")
for key, value in configs:
_run("npm config set %s %s" % (key, shlex.quote(value)))
cmd += " --%s=%s" % (key, shlex.quote(value))

if args:
for key, value in args:
cmd += " --%s=%s" % (key, shlex.quote(value))

return _run(cmd)


class Npm(FetchMethod):
"""Class to fetch a package from a npm registry"""

Expand Down Expand Up @@ -165,14 +189,14 @@ def urldata_init(self, ud, d):

def _resolve_proxy_url(self, ud, d):
def _npm_view():
configs = []
configs.append(("json", "true"))
configs.append(("registry", ud.registry))
args = []
args.append(("json", "true"))
args.append(("registry", ud.registry))
pkgver = shlex.quote(ud.package + "@" + ud.version)
cmd = ud.basecmd + " view %s" % pkgver
env = NpmEnvironment(d)
check_network_access(d, cmd, ud.registry)
view_string = env.run(cmd, configs=configs)
view_string = env.run(cmd, args=args)

if not view_string:
raise FetchError("Unavailable package %s" % pkgver, ud.url)
Expand All @@ -185,8 +209,8 @@ def _npm_view():
raise FetchError(error.get("summary"), ud.url)

if ud.version == "latest":
bb.warn("The npm package %s is using the latest " \
"version available. This could lead to " \
bb.warn("The npm package %s is using the latest "
"version available. This could lead to "
"non-reproducible builds." % pkgver)
elif ud.version != view.get("version"):
raise ParameterError("Invalid 'version' parameter", ud.url)
Expand Down
56 changes: 34 additions & 22 deletions meta/classes/npm.bbclass
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,12 @@ inherit python3native
DEPENDS_prepend = "nodejs-native "
RDEPENDS_${PN}_append_class-target = " nodejs"

EXTRA_OENPM = ""

NPM_INSTALL_DEV ?= "0"

NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}"

def npm_target_arch_map(target_arch):
"""Maps arch names to npm arch names"""
import re
Expand Down Expand Up @@ -57,8 +61,8 @@ def npm_pack(env, srcdir, workdir):
"""Run 'npm pack' on a specified directory"""
import shlex
cmd = "npm pack %s" % shlex.quote(srcdir)
configs = [("ignore-scripts", "true")]
tarball = env.run(cmd, configs=configs, workdir=workdir).strip("\n")
args = [("ignore-scripts", "true")]
tarball = env.run(cmd, args=args, workdir=workdir).strip("\n")
return os.path.join(workdir, tarball)

python npm_do_configure() {
Expand Down Expand Up @@ -132,11 +136,17 @@ python npm_do_configure() {
cached_manifest.pop("dependencies", None)
cached_manifest.pop("devDependencies", None)

with open(orig_shrinkwrap_file, "r") as f:
orig_shrinkwrap = json.load(f)
has_shrinkwrap_file = True

try:
with open(orig_shrinkwrap_file, "r") as f:
orig_shrinkwrap = json.load(f)
except IOError:
has_shrinkwrap_file = False

cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
cached_shrinkwrap.pop("dependencies", None)
if has_shrinkwrap_file:
cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
cached_shrinkwrap.pop("dependencies", None)

# Manage the dependencies
progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
Expand Down Expand Up @@ -167,8 +177,10 @@ python npm_do_configure() {
progress.write("%d/%d" % (progress_done, progress_total))

dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)

if has_shrinkwrap_file:
foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)

# Configure the main package
with tempfile.TemporaryDirectory() as tmpdir:
Expand All @@ -183,16 +195,19 @@ python npm_do_configure() {
cached_manifest[depkey] = {}
cached_manifest[depkey][name] = version

_update_manifest("dependencies")
if has_shrinkwrap_file:
_update_manifest("dependencies")

if dev:
_update_manifest("devDependencies")
if has_shrinkwrap_file:
_update_manifest("devDependencies")

with open(cached_manifest_file, "w") as f:
json.dump(cached_manifest, f, indent=2)

with open(cached_shrinkwrap_file, "w") as f:
json.dump(cached_shrinkwrap, f, indent=2)
if has_shrinkwrap_file:
with open(cached_shrinkwrap_file, "w") as f:
json.dump(cached_shrinkwrap, f, indent=2)
}

python npm_do_compile() {
Expand All @@ -213,15 +228,11 @@ python npm_do_compile() {

bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)

env = NpmEnvironment(d, configs=npm_global_configs(d))

dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)

with tempfile.TemporaryDirectory() as tmpdir:
args = []
configs = []
configs = npm_global_configs(d)

if dev:
if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False):
configs.append(("also", "development"))
else:
configs.append(("only", "production"))
Expand All @@ -236,18 +247,19 @@ python npm_do_compile() {
# Add node-gyp configuration
configs.append(("arch", d.getVar("NPM_ARCH")))
configs.append(("release", "true"))
sysroot = d.getVar("RECIPE_SYSROOT_NATIVE")
nodedir = os.path.join(sysroot, d.getVar("prefix_native").strip("/"))
configs.append(("nodedir", nodedir))
configs.append(("nodedir", d.getVar("NPM_NODEDIR")))
configs.append(("python", d.getVar("PYTHON")))

env = NpmEnvironment(d, configs)

# Add node-pre-gyp configuration
args.append(("target_arch", d.getVar("NPM_ARCH")))
args.append(("build-from-source", "true"))

# Pack and install the main package
tarball = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
env.run("npm install %s" % shlex.quote(tarball), args=args, configs=configs)
cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM"))
env.run(cmd, args=args)
}

npm_do_install() {
Expand Down