Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bazel: Upgrade to 0.23.1 #15

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions BUILD.bazel
Expand Up @@ -8,11 +8,11 @@ package(default_visibility = ["//visibility:public"])
py_binary(
name = "cli",
srcs = ["//bazel_external_data:cli.py"],
visibility = ["//visibility:public"],
deps = [
"//bazel_external_data:core",
"//bazel_external_data:cli_base",
"//bazel_external_data:core",
],
visibility = ["//visibility:public"],
)

# Declared for archive stuff (should generally be internal).
Expand Down
10 changes: 5 additions & 5 deletions bazel_external_data/BUILD.bazel
Expand Up @@ -12,8 +12,8 @@ py_library(
srcs = [
"config_helpers.py",
"core.py",
"util.py",
"hashes.py",
"util.py",
],
imports = imports,
visibility = ["//visibility:public"],
Expand All @@ -22,17 +22,17 @@ py_library(
py_library(
name = "cli_base",
srcs = [
"download.py",
"upload.py",
"check.py",
"download.py",
"squash.py",
"upload.py",
],
imports = imports,
visibility = ["//visibility:public"],
deps = [
":core",
"//bazel_external_data/backends",
],
imports = imports,
visibility = ["//visibility:public"],
)

# Declare 'cli' in //:, to permit easier access to the binary from external
Expand Down
8 changes: 4 additions & 4 deletions bazel_external_data/backends/BUILD.bazel
Expand Up @@ -12,10 +12,10 @@ py_library(
"__init__.py",
"mock.py",
],
imports = imports,
deps = [
"//bazel_external_data:core",
],
imports = imports,
)

py_library(
Expand All @@ -26,21 +26,21 @@ py_library(

py_library(
name = "backends",
imports = imports,
visibility = ["//bazel_external_data:__subpackages__"],
deps = [
":core",
":girder",
],
imports = imports,
visibility = ["//bazel_external_data:__subpackages__"],
)

py_test(
name = "girder_test",
srcs = ["test/girder_test.py"],
tags = ["manual"],
deps = [
":girder",
],
tags = ["manual"],
)

expose_all_files(sub_dirs = ["test"])
13 changes: 9 additions & 4 deletions expose_all_files.bzl
Expand Up @@ -21,7 +21,6 @@ are expandable via `$(locations ...)`.
recursive_filegroup = rule(
attrs = {
"data": attr.label_list(
cfg = "data",
allow_files = True,
mandatory = True,
),
Expand Down Expand Up @@ -64,6 +63,7 @@ def expose_all_files(
@param sub_dirs
Any directories that are not packages.
"""

# @note It'd be nice if this could respect *ignore files, but meh.
# Also, it'd be **super** nice if Bazel did not let `**` globs leak into
# other packages and then error out.
Expand All @@ -76,17 +76,22 @@ def expose_all_files(
srcs = native.glob(patterns)
for sub_dir in sub_dirs:
srcs += native.glob([
sub_dir + "/**/" + pattern for pattern in patterns])
sub_dir + "/**/" + pattern
for pattern in patterns
])
native.filegroup(
name = name,
srcs = srcs,
# Trying to use `data = deps` here only exposes the files in
# runfiles, but not for expansion via `$(locations...)`.
visibility = visibility,
)

# Expose all files recursively (from one level).
deps = [package_prefix + sub_package + ":" + name + "_recursive"
for sub_package in sub_packages]
deps = [
package_prefix + sub_package + ":" + name + "_recursive"
for sub_package in sub_packages
]
recursive_filegroup(
name = name + "_recursive",
data = [name] + deps,
Expand Down
79 changes: 42 additions & 37 deletions external_data.bzl
Expand Up @@ -20,42 +20,43 @@ SETTINGS_DEFAULT = dict(
enable_check_test = True,
)


_HASH_SUFFIX = ".sha512"
_RULE_SUFFIX = "__download"
_RULE_TAG = "external_data"
_TEST_SUFFIX = "__check_test"

# @note This does NOT include 'external_data', so that running with
# --test_tag_filters=external_data does not require a remote.
_TEST_TAGS = ["external_data_check_test"]
_TOOL = "@bazel_external_data_pkg//:cli"
_MANIFEST_SUFFIX = ".manifest.bzl"


def _get_cli_base_args(settings):
args = []

# Argument: Verbosity.
if settings['verbose']:
if settings["verbose"]:
args.append("--verbose")

# Argument: Project root. Guess from the sentinel file rather than PWD, so
# that a file could consumed by a downstream Bazel project.
# (Otherwise, PWD will point to downstream project, which will make a
# conflict.)
args.append("--project_root_guess=$(location {})"
.format(settings['cli_sentinel']))
.format(settings["cli_sentinel"]))

# Extra Arguments (for project settings).
cli_extra_args = settings['cli_extra_args']
cli_extra_args = settings["cli_extra_args"]
if cli_extra_args:
args += cli_extra_args
return args


def external_data(
file,
mode='normal',
settings=SETTINGS_DEFAULT,
tags=[],
visibility=None):
mode = "normal",
settings = SETTINGS_DEFAULT,
tags = [],
visibility = None):
"""Defines an external data file.

@param file
Expand All @@ -68,56 +69,62 @@ def external_data(
Settings for target. See `SETTINGS_DEFAULT` for the each setting and
its purpose.
"""

# Overlay.
# TODO: Check for invalid settings?
settings = SETTINGS_DEFAULT + settings

if mode == 'devel':
if mode == "devel":
# TODO(eric.cousineau): It'd be nice if there is a way to (a) check if
# there is a `*.sha512` file, and if so, (b) check the hash of the
# input file.
if settings['enable_warn']:
if settings["enable_warn"]:
# TODO(eric.cousineau): Print full location of given file?
print("\nexternal_data(file = '{}', mode = 'devel'):".format(file) +
"\n Using local workspace file in development mode." +
"\n Please upload this file and commit the *{} file."
.format(_HASH_SUFFIX))
.format(_HASH_SUFFIX))
native.exports_files(
srcs = [file],
visibility = visibility,
)
elif mode in ['normal', 'no_cache']:
elif mode in ["normal", "no_cache"]:
name = file + _RULE_SUFFIX
hash_file = file + _HASH_SUFFIX

# Binary:
args = ["$(location {})".format(_TOOL)]

# General commands.
args += _get_cli_base_args(settings)

# Subcommand: Download.
args.append("download")

# Argument: Caching.
if mode == 'no_cache':
if mode == "no_cache":
args.append("--no_cache")
else:
# Use symlinking to avoid needing to copy data to sandboxes.
# The cache files are made read-only, so even if a test is run
# with `--spawn_strategy=standalone`, there should be a permission
# error when attempting to write to the file.
args.append("--symlink")

# Argument: Hash file.
args.append("$(location {})".format(hash_file))

# Argument: Output file.
args.append("--output=$@")

cmd = " ".join(args)

if settings['verbose']:
if settings["verbose"]:
print("\nexternal_data(file = '{}', mode = '{}'):"
.format(file, mode) + "\n cmd: {}".format(cmd))
.format(file, mode) + "\n cmd: {}".format(cmd))

cli_sentinel = settings['cli_sentinel']
cli_data = settings['cli_data']
cli_sentinel = settings["cli_sentinel"]
cli_data = settings["cli_data"]
data = [hash_file, cli_sentinel] + cli_data

native.genrule(
Expand All @@ -133,7 +140,7 @@ def external_data(
visibility = visibility,
)

if settings['enable_check_test']:
if settings["enable_check_test"]:
# Add test.
external_data_check_test(
name = file + _TEST_SUFFIX,
Expand All @@ -145,22 +152,21 @@ def external_data(
else:
fail("Invalid mode: {}".format(mode))


def external_data_group(
name,
files,
files_devel = [],
mode='normal',
tags=[],
settings=SETTINGS_DEFAULT,
visibility=None):
mode = "normal",
tags = [],
settings = SETTINGS_DEFAULT,
visibility = None):
"""Defines a group of external data files. """

# Overlay.
settings = SETTINGS_DEFAULT + settings

if settings['enable_warn'] and files_devel and mode == "devel":
print('WARNING: You are specifying `files_devel` and ' +
if settings["enable_warn"] and files_devel and mode == "devel":
print("WARNING: You are specifying `files_devel` and " +
'`mode="devel"`, which is redundant. Try choosing one.')

kwargs = dict(
Expand All @@ -181,7 +187,7 @@ def external_data_group(
if file not in files:
devel_only.append(file)
external_data(file, "devel", **kwargs)
if settings['enable_warn'] and devel_only:
if settings["enable_warn"] and devel_only:
print("""
WARNING: The following `files_devel` files are not in `files`:\n" +
{}
Expand All @@ -198,7 +204,6 @@ WARNING: The following `files_devel` files are not in `files`:\n" +
visibility = visibility,
)


def external_data_check_test(
name,
files,
Expand All @@ -218,8 +223,8 @@ def external_data_check_test(
args = _get_cli_base_args(settings)
args += ["check"] + ["$(location {})".format(x) for x in hash_files]

cli_sentinel = settings['cli_sentinel']
cli_data = settings['cli_data']
cli_sentinel = settings["cli_sentinel"]
cli_data = settings["cli_data"]

# Use `exec.sh` to forward the existing CLI as a test.
# TODO(eric.cousineau): Consider removing "external" as a test tag if it's
Expand All @@ -237,19 +242,17 @@ def external_data_check_test(
)
return name


def get_original_files(hash_files):
"""Gets the original file from a given hash file. """
files = []
for hash_file in hash_files:
if not hash_file.endswith(_HASH_SUFFIX):
fail("Hash file does end with '{}': '{}'"
.format(_HASH_SUFFIX, hash_file))
.format(_HASH_SUFFIX, hash_file))
file = hash_file[:-len(_HASH_SUFFIX)]
files.append(file)
return files


def extract_archive(
name,
manifest,
Expand Down Expand Up @@ -284,6 +287,7 @@ def extract_archive(
@param output_dir
Output directory. If non-empty, must not end with `/`.
"""

# Using: https://groups.google.com/forum/#!topic/bazel-discuss/B5WFlG3co4I
# TODO(eric.cousineau): Add ability to select specific files of the archive
# with globs or something (after stripping prefix).
Expand All @@ -310,8 +314,8 @@ def extract_archive(
fail(("archive: There are no outputs, and empty genrule's are " +
"invalid.\n" +
" After `strip_prefix` filtering, there were no outputs, but " +
"there were {} original files. Did you use the wrong prefix?")
.format(len(manifest["files"])))
"there were {} original files. Did you use the wrong prefix?")
.format(len(manifest["files"])))
else:
output_dir_full = "$(@D)/" + output_dir
tool = "@bazel_external_data_pkg//:extract_archive"
Expand All @@ -334,4 +338,5 @@ def extract_archive(
outs = outs,
tools = [tool],
cmd = cmd,
**kwargs)
**kwargs
)
6 changes: 3 additions & 3 deletions setup/ubuntu/deb_install
Expand Up @@ -41,6 +41,6 @@ EOF
)

dpkg_install_from_wget \
bazel 0.21.0 \
https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel_0.21.0-linux-x86_64.deb \
cdc225dd1c1eb52ac7f4b0bebb40d2c6d6d8bc0f273718b26281077cd70a0403
bazel 0.23.1 \
https://github.com/bazelbuild/bazel/releases/download/0.23.1/bazel_0.23.1-linux-x86_64.deb \
62d7fc733cb64c8bcedec4374e674ffacdc6616584d913fe84b97753c5e0863e
2 changes: 1 addition & 1 deletion test/BUILD.bazel
Expand Up @@ -11,8 +11,8 @@ external_data_workspace_test(

external_data_workspace_test(
name = "workflows_test",
workspace = "external_data_pkg_test",
args = ["./workflows_test.sh"],
workspace = "external_data_pkg_test",
)

exports_files(
Expand Down