diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 45cee7d19..e5b3adc06 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -9,6 +9,7 @@ tasks: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin - bazel run @duplicate_artifacts_test//:pin + - bazel run @regression_testing//:outdated test_flags: - "--//settings:stamp_manifest=True" test_targets: @@ -31,6 +32,7 @@ tasks: shell_commands: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin + - bazel run @regression_testing//:outdated test_targets: - "--" - "//..." @@ -40,6 +42,7 @@ tasks: shell_commands: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin + - bazel run @regression_testing//:outdated test_targets: - "--" - "//..." @@ -49,6 +52,7 @@ tasks: shell_commands: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin + - bazel run @regression_testing//:outdated test_targets: - "--" - "//..." @@ -63,6 +67,7 @@ tasks: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin - bazel run @duplicate_artifacts_test//:pin + - bazel run @regression_testing//:outdated test_targets: - "--" - "//..." @@ -75,6 +80,7 @@ tasks: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin - bazel run @duplicate_artifacts_test//:pin + - bazel run @regression_testing//:outdated test_flags: - "--//settings:stamp_manifest=True" test_targets: @@ -91,6 +97,7 @@ tasks: - bazel run @unpinned_regression_testing//:pin - bazel run @unpinned_maven_install_in_custom_location//:pin - bazel run @duplicate_artifacts_test//:pin + - bazel run @regression_testing//:outdated test_targets: - "--" - "//..." diff --git a/README.md b/README.md index a5c6095f8..06b7c25c6 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Table of Contents * [Custom location for maven_install.json](#custom-location-for-maven_installjson) * [Multiple maven_install.json files](#multiple-maven_installjson-files) * [Generated targets](#generated-targets) + * [Outdated artifacts](#outdated-artifacts) * [Advanced usage](#advanced-usage) * [Fetch source JARs](#fetch-source-jars) * [Checksum verification](#checksum-verification) @@ -285,6 +286,14 @@ the artifact, which integrates with rules like [bazel-common's for generating POM files. See the [`pom_file_generation` example](examples/pom_file_generation/) for more information. +## Outdated artifacts + +To check for updates of artifacts, run the following command at the root of your Bazel workspace: + +``` +$ bazel run @maven//:outdated +``` + ## Advanced usage ### Fetch source JARs @@ -827,7 +836,7 @@ migration, convert legacy Android support library (`com.android.support`) libraries to rely on new AndroidX packages using the [Jetifier](https://developer.android.com/studio/command-line/jetifier) tool. Enable jetification by specifying `jetify = True` in `maven_install.` -Control which artifacts to jetify with `jetify_include_list` — list of artifacts that need to be jetified in `groupId:artifactId` format. +Control which artifacts to jetify with `jetify_include_list` — list of artifacts that need to be jetified in `groupId:artifactId` format. By default all artifacts are jetified if `jetify` is set to True. NOTE: There is a performance penalty to using jetifier due to modifying fetched binaries, fetching @@ -952,7 +961,7 @@ bazel run --stamp \ //user_project:exported_lib.publish` ``` -When using the `gpg_sign` option, the current default key will be used for +When using the `gpg_sign` option, the current default key will be used for signing, and the `gpg` binary needs to be installed on the machine. ## Demo diff --git a/WORKSPACE b/WORKSPACE index 9fce81bb2..408845108 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -51,9 +51,20 @@ http_archive( # dependencies. So, we omit them to keep the WORKSPACE file simpler. # https://skydoc.bazel.build/docs/getting_started_stardoc.html +load("//:defs.bzl", "maven_install") + +maven_install( + name = "outdated", + artifacts = [ + "org.apache.maven:maven-artifact:3.6.3", + ], + repositories = [ + "https://repo1.maven.org/maven2", + ], +) + # Begin test dependencies -load("//:defs.bzl", "maven_install") load("//:specs.bzl", "maven") maven_install( diff --git a/coursier.bzl b/coursier.bzl index 94e954e89..3c9be2e4a 100644 --- a/coursier.bzl +++ b/coursier.bzl @@ -58,6 +58,18 @@ sh_binary( ) """ +_BUILD_OUTDATED = """ +sh_binary( + name = "outdated", + srcs = ["outdated.sh"], + data = [ + "@rules_jvm_external//private/tools/prebuilt:outdated_deploy.jar", + "outdated.artifacts", + "outdated.repositories" + ], +) +""" + def _is_verbose(repository_ctx): return bool(repository_ctx.os.environ.get("RJE_VERBOSE")) @@ -265,6 +277,28 @@ def _get_jq_http_files(): ]) return lines +def _add_outdated_files(repository_ctx, artifacts, repositories): + repository_ctx.file( + "outdated.artifacts", + "\n".join(["{}:{}:{}".format(artifact["group"], artifact["artifact"], artifact["version"]) for artifact in artifacts]) + "\n", + executable = False, + ) + + repository_ctx.file( + "outdated.repositories", + "\n".join([repo["repo_url"] for repo in repositories]) + "\n", + executable = False, + ) + + repository_ctx.template( + "outdated.sh", + repository_ctx.attr._outdated, + { + "{repository_name}": repository_ctx.name, + }, + executable = True, + ) + def _pinned_coursier_fetch_impl(repository_ctx): if not repository_ctx.attr.maven_install_json: fail("Please specify the file label to maven_install.json (e.g." + @@ -272,9 +306,13 @@ def _pinned_coursier_fetch_impl(repository_ctx): _windows_check(repository_ctx) + repositories = [] + for repository in repository_ctx.attr.repositories: + repositories.append(json_parse(repository)) + artifacts = [] - for a in repository_ctx.attr.artifacts: - artifacts.append(json_parse(a)) + for artifact in repository_ctx.attr.artifacts: + artifacts.append(json_parse(artifact)) # Read Coursier state from maven_install.json. repository_ctx.symlink( @@ -396,19 +434,21 @@ def _pinned_coursier_fetch_impl(repository_ctx): "compat_repository.bzl", repository_ctx.attr._compat_repository, substitutions = {}, - executable = False, # not executable + executable = False, ) repository_ctx.file( "BUILD", - _BUILD.format( + (_BUILD + _BUILD_OUTDATED).format( visibility = "private" if repository_ctx.attr.strict_visibility else "public", repository_name = repository_ctx.name, imports = generated_imports, ), - False, # not executable + executable = False, ) + _add_outdated_files(repository_ctx, artifacts, repositories) + # Generate a compatibility layer of external repositories for all jar artifacts. if repository_ctx.attr.generate_compat_repositories: compat_repositories_bzl = ["load(\"@%s//:compat_repository.bzl\", \"compat_repository\")" % repository_ctx.name] @@ -423,7 +463,7 @@ def _pinned_coursier_fetch_impl(repository_ctx): repository_ctx.file( "compat.bzl", "\n".join(compat_repositories_bzl) + "\n", - False, # not executable + executable = False, ) def split_url(url): @@ -644,12 +684,12 @@ def _coursier_fetch_impl(repository_ctx): repositories.append(json_parse(repository)) artifacts = [] - for a in repository_ctx.attr.artifacts: - artifacts.append(json_parse(a)) + for artifact in repository_ctx.attr.artifacts: + artifacts.append(json_parse(artifact)) excluded_artifacts = [] - for a in repository_ctx.attr.excluded_artifacts: - excluded_artifacts.append(json_parse(a)) + for artifact in repository_ctx.attr.excluded_artifacts: + excluded_artifacts.append(json_parse(artifact)) # Once coursier finishes a fetch, it generates a tree of artifacts and their # transitive dependencies in a JSON file. We use that as the source of truth @@ -789,7 +829,7 @@ def _coursier_fetch_impl(repository_ctx): repository_ctx.file( "hasher_argsfile", "\n".join([str(f) for f in files_to_hash]) + "\n", - False, # Not executable + executable = False, ) exec_result = repository_ctx.execute( hasher_command + ["--argsfile", repository_ctx.path("hasher_argsfile")], @@ -840,15 +880,17 @@ def _coursier_fetch_impl(repository_ctx): repository_name = repository_ctx.name[len("unpinned_"):] else: repository_name = repository_ctx.name + # Add outdated artifact files if this is a pinned repo + _add_outdated_files(repository_ctx, artifacts, repositories) repository_ctx.file( "BUILD", - (_BUILD + _BUILD_PIN).format( + (_BUILD + _BUILD_PIN + _BUILD_OUTDATED).format( visibility = "private" if repository_ctx.attr.strict_visibility else "public", repository_name = repository_name, imports = generated_imports, ), - False, # not executable + executable = False, ) # If maven_install.json has already been used in maven_install, @@ -906,7 +948,7 @@ def _coursier_fetch_impl(repository_ctx): "compat_repository.bzl", repository_ctx.attr._compat_repository, substitutions = {}, - executable = False, # not executable + executable = False, ) compat_repositories_bzl = ["load(\"@%s//:compat_repository.bzl\", \"compat_repository\")" % repository_ctx.name] @@ -921,12 +963,14 @@ def _coursier_fetch_impl(repository_ctx): repository_ctx.file( "compat.bzl", "\n".join(compat_repositories_bzl) + "\n", - False, # not executable + executable = False, ) pinned_coursier_fetch = repository_rule( attrs = { "_compat_repository": attr.label(default = "//:private/compat_repository.bzl"), + "_outdated": attr.label(default = "//:private/outdated.sh"), + "repositories": attr.string_list(), # list of repository objects, each as json "artifacts": attr.string_list(), # list of artifact objects, each as json "fetch_sources": attr.bool(default = False), "generate_compat_repositories": attr.bool(default = False), # generate a compatible layer with repositories for each artifact @@ -952,6 +996,7 @@ coursier_fetch = repository_rule( "_sha256_hasher": attr.label(default = "//private/tools/prebuilt:hasher_deploy.jar"), "_pin": attr.label(default = "//:private/pin.sh"), "_compat_repository": attr.label(default = "//:private/compat_repository.bzl"), + "_outdated": attr.label(default = "//:private/outdated.sh"), "repositories": attr.string_list(), # list of repository objects, each as json "artifacts": attr.string_list(), # list of artifact objects, each as json "fail_on_missing_checksum": attr.bool(default = True), diff --git a/defs.bzl b/defs.bzl index e016ed977..b0f7dac28 100644 --- a/defs.bzl +++ b/defs.bzl @@ -126,6 +126,7 @@ def maven_install( # Create the repository generated from a maven_install.json file. pinned_coursier_fetch( name = name, + repositories = repositories_json_strings, artifacts = artifacts_json_strings, maven_install_json = maven_install_json, fetch_sources = fetch_sources, diff --git a/private/outdated.sh b/private/outdated.sh new file mode 100644 index 000000000..489b1fe15 --- /dev/null +++ b/private/outdated.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ -f "private/tools/prebuilt/outdated_deploy.jar" ]; then + java -jar private/tools/prebuilt/outdated_deploy.jar external/{repository_name}/outdated.artifacts external/{repository_name}/outdated.repositories +else + java -jar external/rules_jvm_external/private/tools/prebuilt/outdated_deploy.jar external/{repository_name}/outdated.artifacts external/{repository_name}/outdated.repositories +fi diff --git a/private/tools/java/rules/jvm/external/maven/BUILD.bazel b/private/tools/java/rules/jvm/external/maven/BUILD.bazel index 37e9f5579..f0da24743 100644 --- a/private/tools/java/rules/jvm/external/maven/BUILD.bazel +++ b/private/tools/java/rules/jvm/external/maven/BUILD.bazel @@ -8,8 +8,24 @@ java_binary( "8", ], main_class = "rules.jvm.external.maven.MavenPublisher", + visibility = ["//visibility:public"], deps = [ "//private/tools/java/rules/jvm/external:byte-streams", ], +) + +java_binary( + name = "outdated", + srcs = ["Outdated.java"], + javacopts = [ + "-source", + "8", + "-target", + "8", + ], + main_class = "rules.jvm.external.maven.Outdated", visibility = ["//visibility:public"], + deps = [ + "@outdated//:org_apache_maven_maven_artifact", + ], ) diff --git a/private/tools/java/rules/jvm/external/maven/Outdated.java b/private/tools/java/rules/jvm/external/maven/Outdated.java new file mode 100644 index 000000000..ff5cdfdf7 --- /dev/null +++ b/private/tools/java/rules/jvm/external/maven/Outdated.java @@ -0,0 +1,126 @@ +package rules.jvm.external.maven; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.apache.maven.artifact.versioning.ComparableVersion; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +public class Outdated { + public static String getReleaseVersion(String repository, String groupId, String artifactId) { + String releaseVersion = null; + + String url = + String.format("%s/%s/%s/maven-metadata.xml", + repository, + groupId.replaceAll("\\.", "/"), + artifactId); + + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder documentBuilder; + try { + documentBuilder = documentBuilderFactory.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + verboseLog(String.format("Caught exception %s", e)); + return null; + } + Document document; + try { + document = documentBuilder.parse(new URL(url).openStream()); + } catch (IOException | SAXException e) { + verboseLog(String.format("Caught exception for %s: %s", url, e)); + return null; + } + + // example maven-metadata.xml + // + // + // 1.14.0-SNAPSHOT + // 1.13.0 + // + // + Element metadataElement = document.getDocumentElement(); + Element versioningElement = getFirstChildElement(metadataElement, "versioning"); + if (versioningElement != null) { + releaseVersion = versioningElement.getElementsByTagName("release").item(0).getTextContent(); + } else { + verboseLog(String.format("Could not find tag for %s, returning null version", url)); + } + return releaseVersion; + } + + public static Element getFirstChildElement(Element element, String tagName) { + NodeList nodeList = element.getElementsByTagName(tagName); + for (int i = 0; i < nodeList.getLength(); i++) + { + Node node = nodeList.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE) + { + return (Element) node; + } + } + return null; + } + + public static void verboseLog(String logline) { + if (System.getenv("RJE_VERBOSE") != null) { + System.out.println(logline); + } + } + + public static void main(String[] args) throws IOException { + verboseLog(String.format("Running outdated with args %s", Arrays.toString(args))); + + if (args.length != 2) { + System.out.println("Usage: outdated artifacts = Files.readAllLines(Paths.get(artifactsFilePath), StandardCharsets.UTF_8); + List repositories = Files.readAllLines(Paths.get(repositoriesFilePath), StandardCharsets.UTF_8); + + System.out.println(String.format("Checking for updates of %d artifacts against the following repositories:", artifacts.size())); + for (String repository: repositories) { + System.out.println(String.format("\t%s", repository)); + } + System.out.println(); + + for (String artifact: artifacts) { + String[] artifactParts = artifact.split(":"); + String groupId = artifactParts[0]; + String artifactId = artifactParts[1]; + String version = artifactParts[2]; + + String releaseVersion = null; + for (String repository : repositories) { + releaseVersion = getReleaseVersion(repository, groupId, artifactId); + if (releaseVersion != null) { + verboseLog(String.format("Found version [%s] for %s:%s in %s", releaseVersion, groupId, artifactId, repository)); + // Should we search all repositories in the list for latest version instead of just the first + // repository that has a version? + break; + } + } + + if (releaseVersion == null) { + verboseLog(String.format("Could not find version for %s:%s", groupId, artifactId)); + } else if (new ComparableVersion(releaseVersion).compareTo(new ComparableVersion(version)) > 0) { + System.out.println(String.format("%s:%s [%s -> %s]", groupId, artifactId, version, releaseVersion)); + } + } + } +} diff --git a/private/tools/prebuilt/BUILD b/private/tools/prebuilt/BUILD index 72f47b0f8..00647a41c 100644 --- a/private/tools/prebuilt/BUILD +++ b/private/tools/prebuilt/BUILD @@ -1,5 +1,6 @@ # This package contains static executables used by the maven_install repository rule. exports_files([ - "hasher_deploy.jar", # built from //tools/java:hasher_deploy.jar + "hasher_deploy.jar", # built from //private/tools/java:hasher_deploy.jar + "outdated_deploy.jar", # built from //private/tools/java/rules/jvm/external/maven:outdated_deploy.jar ]) diff --git a/private/tools/prebuilt/outdated_deploy.jar b/private/tools/prebuilt/outdated_deploy.jar new file mode 100755 index 000000000..3dfb263c5 Binary files /dev/null and b/private/tools/prebuilt/outdated_deploy.jar differ diff --git a/third_party/README.md b/third_party/bazel_json/README.md similarity index 69% rename from third_party/README.md rename to third_party/bazel_json/README.md index 5dcff04a9..48f5fe0b5 100644 --- a/third_party/README.md +++ b/third_party/bazel_json/README.md @@ -1,6 +1,4 @@ -# Vendored third_party dependencies - -## bazel_json +# bazel_json - Source: https://github.com/erickj/bazel_json - Commit: e954ef2c28cd92d97304810e8999e1141e2b5cc8