Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JVM lockfile invalidation headers and verification #14185

Merged
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
e5142c3
Adds `JVMLockfileMetadata` and related things
Jan 12, 2022
a274adf
Moves `CoursierResolvedLockfile.from_json_dict` to `.from_serialized`…
Jan 12, 2022
60b3f43
Serializes coursier lockfiles to toml, and adds initial support for r…
Jan 13, 2022
8595016
Generated lockfiles now include invalidation headers
Jan 13, 2022
1536ac9
WIP
Jan 13, 2022
df0c578
Pulls `Coordinate` and `ArtifactRequirements` out into a separate fil…
Jan 13, 2022
3ceac13
Pulls `Coordinate` and `ArtifactRequirements` out into a separate fil…
Jan 13, 2022
31cc4b7
Validate lockfiles in `jvm_tool.py`
Jan 14, 2022
53ffa56
Regenerates tool lockfiles into new toml format
Jan 14, 2022
89b6f18
Validates lockfile headers for google java format
Jan 14, 2022
37c464c
All JVM tools validate their lockfile on setup
Jan 14, 2022
4129ee0
Merge remote-tracking branch 'origin/main' into chrisjrn/13373-jvm-lo…
Jan 14, 2022
0e83cf1
Updates JVM user lockfiles
Jan 14, 2022
b574265
Merge branch 'main' into chrisjrn/13373-jvm-lockfile-invalidation
Jan 18, 2022
49e23b3
Fix the tool lockfile generating code to actually execute
Jan 18, 2022
3c50573
fmt
Jan 18, 2022
3434461
Fix tests
Jan 18, 2022
410c115
Add validation of user lockfiles
Jan 18, 2022
2a43202
Fix tests
Jan 18, 2022
9a6d8fb
Fixmore tests
Jan 18, 2022
622332f
fmt
Jan 18, 2022
65af690
Renames `TCoursierResolvedLockfile` to `CoursierLockfileWrapper`
Jan 19, 2022
647c9f5
Update error message to be less terse/useless
Jan 19, 2022
e987f2e
Updates `is_valid_for` docstring
Jan 19, 2022
fc53b02
simplifies junit_test slightly
Jan 19, 2022
b55f229
fix parenthesisation
Jan 19, 2022
0dc2130
Merge remote-tracking branch 'origin/main' into chrisjrn/13373-jvm-lo…
Jan 19, 2022
858f83e
Address code review floop
Jan 19, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 11 additions & 0 deletions 3rdparty/jvm/global_scalac_plugins.lock
@@ -1 +1,12 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": []
# }
# --- END PANTS LOCKFILE METADATA ---
Eric-Arellano marked this conversation as resolved.
Show resolved Hide resolved

entries = []
11 changes: 11 additions & 0 deletions 3rdparty/jvm/testprojects.lockfile
@@ -1 +1,12 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": []
# }
# --- END PANTS LOCKFILE METADATA ---

entries = []
@@ -1,3 +1,16 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": [
# "org.apache.avro:avro-tools:1.11.0,url=not_provided,jar=not_provided"
# ]
# }
# --- END PANTS LOCKFILE METADATA ---

[[entries]]
directDependencies = []
dependencies = []
Expand Down
11 changes: 9 additions & 2 deletions src/python/pants/backend/codegen/avro/java/rules.py
Expand Up @@ -37,7 +37,12 @@
from pants.jvm.goals import lockfile
from pants.jvm.goals.lockfile import JvmLockfileRequest
from pants.jvm.jdk_rules import JdkSetup
from pants.jvm.resolve.coursier_fetch import MaterializedClasspath, MaterializedClasspathRequest
from pants.jvm.resolve.coursier_fetch import (
CoursierResolvedLockfile,
MaterializedClasspath,
MaterializedClasspathRequest,
)
from pants.jvm.resolve.jvm_tool import ValidatedJvmToolLockfileRequest
from pants.source.source_root import SourceRoot, SourceRootRequest
from pants.util.logging import LogLevel

Expand Down Expand Up @@ -98,11 +103,13 @@ async def compile_avro_source(
output_dir = "_generated_files"
toolcp_relpath = "__toolcp"

lockfile = await Get(CoursierResolvedLockfile, ValidatedJvmToolLockfileRequest(avro_tools))

tool_classpath, subsetted_input_digest, empty_output_dir = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
lockfiles=(avro_tools.resolved_lockfile(),),
lockfiles=(lockfile,),
),
),
Get(
Expand Down
16 changes: 12 additions & 4 deletions src/python/pants/backend/codegen/protobuf/scala/rules.py
Expand Up @@ -48,8 +48,12 @@
from pants.jvm.goals.lockfile import JvmLockfileRequest
from pants.jvm.jdk_rules import JdkSetup
from pants.jvm.resolve.common import ArtifactRequirements, Coordinate
from pants.jvm.resolve.coursier_fetch import MaterializedClasspath, MaterializedClasspathRequest
from pants.jvm.resolve.jvm_tool import GatherJvmCoordinatesRequest
from pants.jvm.resolve.coursier_fetch import (
CoursierResolvedLockfile,
MaterializedClasspath,
MaterializedClasspathRequest,
)
from pants.jvm.resolve.jvm_tool import GatherJvmCoordinatesRequest, ValidatedJvmToolLockfileRequest
from pants.source.source_root import SourceRoot, SourceRootRequest
from pants.util.logging import LogLevel
from pants.util.ordered_set import FrozenOrderedSet
Expand Down Expand Up @@ -112,6 +116,8 @@ async def generate_scala_from_protobuf(
plugins_relpath = "__plugins"
protoc_relpath = "__protoc"

lockfile = await Get(CoursierResolvedLockfile, ValidatedJvmToolLockfileRequest(scalapb))

(
downloaded_protoc_binary,
tool_classpath,
Expand All @@ -123,7 +129,7 @@ async def generate_scala_from_protobuf(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
lockfiles=(scalapb.resolved_lockfile(),),
lockfiles=(lockfile,),
),
),
Get(Digest, CreateDigest([Directory(output_dir)])),
Expand Down Expand Up @@ -279,6 +285,8 @@ async def setup_scalapb_shim_classfiles(

scalapb_shim_source = FileContent("ScalaPBShim.scala", scalapb_shim_content)

lockfile = await Get(CoursierResolvedLockfile, ValidatedJvmToolLockfileRequest(scalapb))

tool_classpath, shim_classpath, source_digest = await MultiGet(
Get(
MaterializedClasspath,
Expand Down Expand Up @@ -311,7 +319,7 @@ async def setup_scalapb_shim_classfiles(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__shimcp",
lockfiles=(scalapb.resolved_lockfile(),),
lockfiles=(lockfile,),
),
),
Get(
Expand Down
@@ -1,3 +1,16 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": [
# "com.thesamet.scalapb:scalapbc_2.13:0.11.6,url=not_provided,jar=not_provided"
# ]
# }
# --- END PANTS LOCKFILE METADATA ---

[[entries]]
file_name = "com.chuusai_shapeless_2.13_2.3.3.jar"
[[entries.directDependencies]]
Expand Down
11 changes: 9 additions & 2 deletions src/python/pants/backend/codegen/thrift/scrooge/rules.py
Expand Up @@ -18,7 +18,12 @@
from pants.jvm.goals import lockfile
from pants.jvm.goals.lockfile import JvmLockfileRequest
from pants.jvm.jdk_rules import JdkSetup
from pants.jvm.resolve.coursier_fetch import MaterializedClasspath, MaterializedClasspathRequest
from pants.jvm.resolve.coursier_fetch import (
CoursierResolvedLockfile,
MaterializedClasspath,
MaterializedClasspathRequest,
)
from pants.jvm.resolve.jvm_tool import ValidatedJvmToolLockfileRequest
from pants.source.source_root import SourceRootsRequest, SourceRootsResult
from pants.util.logging import LogLevel

Expand Down Expand Up @@ -49,11 +54,13 @@ async def generate_scrooge_thrift_sources(
output_dir = "_generated_files"
toolcp_relpath = "__toolcp"

lockfile = await Get(CoursierResolvedLockfile, ValidatedJvmToolLockfileRequest(scrooge))

tool_classpath, transitive_targets, empty_output_dir_digest, wrapped_target = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
lockfiles=(scrooge.resolved_lockfile(),),
lockfiles=(lockfile,),
),
),
Get(TransitiveTargets, TransitiveTargetsRequest([request.thrift_source_field.address])),
Expand Down
@@ -1,3 +1,16 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": [
# "com.twitter:scrooge-generator_2.13:21.12.0,url=not_provided,jar=not_provided"
# ]
# }
# --- END PANTS LOCKFILE METADATA ---

[[entries]]
file_name = "com.github.scopt_scopt_2.13_4.0.0-RC2.jar"
[[entries.directDependencies]]
Expand Down
50 changes: 16 additions & 34 deletions src/python/pants/backend/java/compile/javac_test.py
Expand Up @@ -25,10 +25,11 @@
from pants.jvm import jdk_rules, testutil
from pants.jvm.compile import ClasspathEntry, CompileResult, FallibleClasspathEntry
from pants.jvm.goals.coursier import rules as coursier_rules
from pants.jvm.resolve.common import Coordinate, Coordinates
from pants.jvm.resolve.common import ArtifactRequirement, Coordinate, Coordinates
from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry, CoursierResolvedLockfile
from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules
from pants.jvm.resolve.coursier_test_util import TCoursierResolvedLockfile
from pants.jvm.target_types import JvmArtifactTarget
from pants.jvm.testutil import (
RenderedClasspath,
Expand Down Expand Up @@ -103,9 +104,7 @@ def test_compile_no_deps(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"BUILD": "java_sources(name='lib')",
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"ExampleLib.java": JAVA_LIB_SOURCE,
}
)
Expand Down Expand Up @@ -145,9 +144,7 @@ def test_compile_jdk_versions(rule_runner: RuleRunner) -> None:
)
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"ExampleLib.java": JAVA_LIB_SOURCE,
}
)
Expand Down Expand Up @@ -182,9 +179,7 @@ def test_compile_multiple_source_files(rule_runner: RuleRunner) -> None:
)
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"ExampleLib.java": JAVA_LIB_SOURCE,
"OtherLib.java": dedent(
"""\
Expand Down Expand Up @@ -262,9 +257,7 @@ def test_compile_with_cycle(rule_runner: RuleRunner) -> None:
"""\
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"a/BUILD": dedent(
"""\
java_sources(
Expand Down Expand Up @@ -350,9 +343,7 @@ def test_compile_with_transitive_cycle(rule_runner: RuleRunner) -> None:
public class Main implements A {}
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"a/BUILD": dedent(
"""\
java_sources(
Expand Down Expand Up @@ -436,9 +427,7 @@ def test_compile_with_transitive_multiple_sources(rule_runner: RuleRunner) -> No
class Other implements B {}
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"lib/BUILD": dedent(
"""\
java_sources(
Expand Down Expand Up @@ -497,9 +486,7 @@ def test_compile_with_deps(rule_runner: RuleRunner) -> None:
)
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"Example.java": JAVA_LIB_MAIN_SOURCE,
"lib/BUILD": dedent(
"""\
Expand Down Expand Up @@ -540,9 +527,7 @@ def test_compile_of_package_info(rule_runner: RuleRunner) -> None:
)
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"package-info.java": dedent(
"""
package org.pantsbuild.example;
Expand Down Expand Up @@ -581,9 +566,7 @@ def test_compile_with_missing_dep_fails(rule_runner: RuleRunner) -> None:
"""
),
"Example.java": JAVA_LIB_MAIN_SOURCE,
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
}
)
request = CompileJavaSourceRequest(
Expand All @@ -599,10 +582,11 @@ def test_compile_with_missing_dep_fails(rule_runner: RuleRunner) -> None:

@maybe_skip_jdk_test
def test_compile_with_maven_deps(rule_runner: RuleRunner) -> None:
resolved_joda_lockfile = CoursierResolvedLockfile(
joda_coord = Coordinate(group="joda-time", artifact="joda-time", version="2.10.10")
resolved_joda_lockfile = TCoursierResolvedLockfile.new(
entries=(
CoursierLockfileEntry(
coord=Coordinate(group="joda-time", artifact="joda-time", version="2.10.10"),
coord=joda_coord,
file_name="joda-time-2.10.10.jar",
direct_dependencies=Coordinates([]),
dependencies=Coordinates([]),
Expand Down Expand Up @@ -633,7 +617,7 @@ def test_compile_with_maven_deps(rule_runner: RuleRunner) -> None:
)
"""
),
"3rdparty/jvm/default.lock": resolved_joda_lockfile.to_serialized().decode(),
"3rdparty/jvm/default.lock": resolved_joda_lockfile.serialize([ArtifactRequirement(coordinate=joda_coord)]),
"Example.java": dedent(
"""
package org.pantsbuild.example;
Expand Down Expand Up @@ -674,9 +658,7 @@ def test_compile_with_missing_maven_dep_fails(rule_runner: RuleRunner) -> None:
)
"""
),
"3rdparty/jvm/default.lock": CoursierResolvedLockfile(entries=())
.to_serialized()
.decode(),
"3rdparty/jvm/default.lock": TCoursierResolvedLockfile.new(entries=()).serialize(),
"Example.java": dedent(
"""
package org.pantsbuild.example;
Expand Down
@@ -1,3 +1,18 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": [
# "com.fasterxml.jackson.core:jackson-databind:2.12.4,url=not_provided,jar=not_provided",
# "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.12.4,url=not_provided,jar=not_provided",
# "com.github.javaparser:javaparser-symbol-solver-core:3.23.0,url=not_provided,jar=not_provided"
# ]
# }
# --- END PANTS LOCKFILE METADATA ---

[[entries]]
directDependencies = []
dependencies = []
Expand Down
@@ -1,3 +1,16 @@
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants generate-lockfiles
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 1,
# "generated_with_requirements": [
# "com.google.googlejavaformat:google-java-format:1.13.0,url=not_provided,jar=not_provided"
# ]
# }
# --- END PANTS LOCKFILE METADATA ---

[[entries]]
directDependencies = []
dependencies = []
Expand Down