Add protobuf linting using Buf with `pants.backend.codegen.protobuf.l…

…int.buf` backend (#14540)

Closes #13189.

[ci skip-rust]

build-support/bin/generate_docs.py

@@ -203,6 +203,7 @@ def run_pants_help_all() -> dict[str, Any]:
     # List all (stable enough) backends here.
     backends = [
         "pants.backend.awslambda.python",
+        "pants.backend.codegen.protobuf.lint.buf",
         "pants.backend.codegen.protobuf.python",
         "pants.backend.codegen.thrift.apache.python",
         "pants.backend.docker",

src/python/pants/backend/codegen/protobuf/lint/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/codegen/protobuf/lint/buf/BUILD

@@ -0,0 +1,5 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()
+python_tests(name="tests")

src/python/pants/backend/codegen/protobuf/lint/buf/register.py

@@ -0,0 +1,9 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.codegen.protobuf.lint.buf import skip_field
+from pants.backend.codegen.protobuf.lint.buf.rules import rules as buf_rules
+
+
+def rules():
+    return (*buf_rules(), *skip_field.rules())

src/python/pants/backend/codegen/protobuf/lint/buf/rules.py

@@ -0,0 +1,109 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from dataclasses import dataclass
+
+from pants.backend.codegen.protobuf.lint.buf.skip_field import SkipBufField
+from pants.backend.codegen.protobuf.lint.buf.subsystem import BufSubsystem
+from pants.backend.codegen.protobuf.target_types import (
+    ProtobufDependenciesField,
+    ProtobufSourceField,
+)
+from pants.core.goals.lint import LintResult, LintResults, LintTargetsRequest
+from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest
+from pants.core.util_rules.source_files import SourceFilesRequest
+from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
+from pants.engine.fs import Digest, MergeDigests
+from pants.engine.platform import Platform
+from pants.engine.process import FallibleProcessResult, Process
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.target import FieldSet, Target, TransitiveTargets, TransitiveTargetsRequest
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+from pants.util.strutil import pluralize
+
+
+@dataclass(frozen=True)
+class BufFieldSet(FieldSet):
+    required_fields = (ProtobufSourceField,)
+
+    sources: ProtobufSourceField
+    dependencies: ProtobufDependenciesField
+
+    @classmethod
+    def opt_out(cls, tgt: Target) -> bool:
+        return tgt.get(SkipBufField).value
+
+
+class BufRequest(LintTargetsRequest):
+    field_set_type = BufFieldSet
+    name = BufSubsystem.options_scope
+
+
+@rule(desc="Lint with Buf", level=LogLevel.DEBUG)
+async def run_buf(request: BufRequest, buf: BufSubsystem) -> LintResults:
+    if buf.skip:
+        return LintResults([], linter_name=request.name)
+
+    transitive_targets = await Get(
+        TransitiveTargets,
+        TransitiveTargetsRequest((field_set.address for field_set in request.field_sets)),
+    )
+
+    all_stripped_sources_request = Get(
+        StrippedSourceFiles,
+        SourceFilesRequest(
+            tgt[ProtobufSourceField]
+            for tgt in transitive_targets.closure
+            if tgt.has_field(ProtobufSourceField)
+        ),
+    )
+    target_stripped_sources_request = Get(
+        StrippedSourceFiles,
+        SourceFilesRequest(
+            (field_set.sources for field_set in request.field_sets),
+            for_sources_types=(ProtobufSourceField,),
+            enable_codegen=True,
+        ),
+    )
+
+    download_buf_get = Get(
+        DownloadedExternalTool, ExternalToolRequest, buf.get_request(Platform.current)
+    )
+
+    target_sources_stripped, all_sources_stripped, downloaded_buf = await MultiGet(
+        target_stripped_sources_request, all_stripped_sources_request, download_buf_get
+    )
+
+    input_digest = await Get(
+        Digest,
+        MergeDigests(
+            (
+                target_sources_stripped.snapshot.digest,
+                all_sources_stripped.snapshot.digest,
+                downloaded_buf.digest,
+            )
+        ),
+    )
+
+    process_result = await Get(
+        FallibleProcessResult,
+        Process(
+            argv=[
+                downloaded_buf.exe,
+                "lint",
+                *buf.args,
+                "--path",
+                ",".join(target_sources_stripped.snapshot.files),
+            ],
+            input_digest=input_digest,
+            description=f"Run Buf on {pluralize(len(request.field_sets), 'file')}.",
+            level=LogLevel.DEBUG,
+        ),
+    )
+    result = LintResult.from_fallible_process_result(process_result)
+
+    return LintResults([result], linter_name=request.name)
+
+
+def rules():
+    return [*collect_rules(), UnionRule(LintTargetsRequest, BufRequest)]

src/python/pants/backend/codegen/protobuf/lint/buf/rules_integration_test.py

@@ -0,0 +1,200 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import json
+from textwrap import dedent
+
+import pytest
+
+from pants.backend.codegen.protobuf.lint.buf.rules import BufFieldSet, BufRequest
+from pants.backend.codegen.protobuf.lint.buf.rules import rules as buf_rules
+from pants.backend.codegen.protobuf.target_types import ProtobufSourcesGeneratorTarget
+from pants.backend.codegen.protobuf.target_types import rules as target_types_rules
+from pants.core.goals.lint import LintResult, LintResults
+from pants.core.util_rules import config_files, external_tool, stripped_source_files
+from pants.engine.addresses import Address
+from pants.engine.target import Target
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *buf_rules(),
+            *config_files.rules(),
+            *external_tool.rules(),
+            *stripped_source_files.rules(),
+            *target_types_rules(),
+            QueryRule(LintResults, [BufRequest]),
+        ],
+        target_types=[ProtobufSourcesGeneratorTarget],
+    )
+
+
+GOOD_FILE = 'syntax = "proto3";\npackage foo.v1;\nmessage Foo {\nstring snake_case = 1;\n}\n'
+BAD_FILE = 'syntax = "proto3";\npackage foo.v1;\nmessage Bar {\nstring camelCase = 1;\n}\n'
+
+
+def run_buf(
+    rule_runner: RuleRunner,
+    targets: list[Target],
+    *,
+    source_roots: list[str] | None = None,
+    extra_args: list[str] | None = None,
+) -> tuple[LintResult, ...]:
+    rule_runner.set_options(
+        [
+            f"--source-root-patterns={repr(source_roots)}",
+            "--backend-packages=pants.backend.codegen.protobuf.lint.buf",
+            *(extra_args or ()),
+        ],
+        env_inherit={"PATH"},
+    )
+    results = rule_runner.request(
+        LintResults,
+        [BufRequest(BufFieldSet.create(tgt) for tgt in targets)],
+    )
+    return results.results
+
+
+def assert_success(
+    rule_runner: RuleRunner,
+    target: Target,
+    *,
+    source_roots: list[str] | None = None,
+    extra_args: list[str] | None = None,
+) -> None:
+    result = run_buf(rule_runner, [target], source_roots=source_roots, extra_args=extra_args)
+    assert len(result) == 1
+    assert result[0].exit_code == 0
+    assert not result[0].stdout
+    assert not result[0].stderr
+
+
+def test_passing(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {"foo/v1/f.proto": GOOD_FILE, "foo/v1/BUILD": "protobuf_sources(name='t')"}
+    )
+    tgt = rule_runner.get_target(Address("foo/v1", target_name="t", relative_file_path="f.proto"))
+    assert_success(rule_runner, tgt)
+
+
+def test_failing(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {"foo/v1/f.proto": BAD_FILE, "foo/v1/BUILD": "protobuf_sources(name='t')"}
+    )
+    tgt = rule_runner.get_target(Address("foo/v1", target_name="t", relative_file_path="f.proto"))
+    result = run_buf(rule_runner, [tgt])
+    assert len(result) == 1
+    assert result[0].exit_code == 100
+    assert "foo/v1/f.proto:" in result[0].stdout
+
+
+def test_multiple_targets(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "foo/v1/good.proto": GOOD_FILE,
+            "foo/v1/bad.proto": BAD_FILE,
+            "foo/v1/BUILD": "protobuf_sources(name='t')",
+        }
+    )
+    tgts = [
+        rule_runner.get_target(Address("foo/v1", target_name="t", relative_file_path="good.proto")),
+        rule_runner.get_target(Address("foo/v1", target_name="t", relative_file_path="bad.proto")),
+    ]
+    result = run_buf(rule_runner, tgts)
+    assert len(result) == 1
+    assert result[0].exit_code == 100
+    assert "good.proto" not in result[0].stdout
+    assert "foo/v1/bad.proto:" in result[0].stdout
+
+
+def test_passthrough_args(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {"foo/v1/f.proto": BAD_FILE, "foo/v1/BUILD": "protobuf_sources(name='t')"}
+    )
+    tgt = rule_runner.get_target(Address("foo/v1", target_name="t", relative_file_path="f.proto"))
+    config = json.dumps(
+        {
+            "version": "v1",
+            "lint": {
+                "ignore_only": {
+                    "FIELD_LOWER_SNAKE_CASE": [
+                        "foo/v1/f.proto",
+                    ],
+                },
+            },
+        }
+    )
+
+    assert_success(
+        rule_runner,
+        tgt,
+        extra_args=[f"--buf-lint-args='--config={config}'"],
+    )
+
+
+def test_skip(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {"foo/v1/f.proto": BAD_FILE, "foo/v1/BUILD": "protobuf_sources(name='t')"}
+    )
+    tgt = rule_runner.get_target(Address("foo/v1", target_name="t", relative_file_path="f.proto"))
+    result = run_buf(rule_runner, [tgt], extra_args=["--buf-lint-skip"])
+    assert not result
+
+
+def test_dependencies(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "src/protobuf/dir1/v1/f.proto": dedent(
+                """\
+                syntax = "proto3";
+                package dir1.v1;
+                message Person {
+                  string name = 1;
+                  int32 id = 2;
+                  string email = 3;
+                }
+                """
+            ),
+            "src/protobuf/dir1/v1/BUILD": "protobuf_sources()",
+            "src/protobuf/dir2/v1/f.proto": dedent(
+                """\
+                syntax = "proto3";
+                package dir2.v1;
+                import "dir1/v1/f.proto";
+
+                message Person {
+                  dir1.v1.Person person = 1;
+                }
+                """
+            ),
+            "src/protobuf/dir2/v1/BUILD": (
+                "protobuf_sources(dependencies=['src/protobuf/dir1/v1'])"
+            ),
+            "tests/protobuf/test_protos/v1/f.proto": dedent(
+                """\
+                syntax = "proto3";
+                package test_protos.v1;
+                import "dir2/v1/f.proto";
+
+                message Person {
+                  dir2.v1.Person person = 1;
+                }
+                """
+            ),
+            "tests/protobuf/test_protos/v1/BUILD": (
+                "protobuf_sources(dependencies=['src/protobuf/dir2/v1'])"
+            ),
+        }
+    )
+
+    tgt = rule_runner.get_target(
+        Address("tests/protobuf/test_protos/v1", relative_file_path="f.proto")
+    )
+    assert_success(
+        rule_runner, tgt, source_roots=["src/python", "/src/protobuf", "/tests/protobuf"]
+    )

src/python/pants/backend/codegen/protobuf/lint/buf/skip_field.py

@@ -0,0 +1,21 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.codegen.protobuf.target_types import (
+    ProtobufSourcesGeneratorTarget,
+    ProtobufSourceTarget,
+)
+from pants.engine.target import BoolField
+
+
+class SkipBufField(BoolField):
+    alias = "skip_buf_lint"
+    default = False
+    help = "If true, don't lint this target's code with Buf."
+
+
+def rules():
+    return [
+        ProtobufSourceTarget.register_plugin_field(SkipBufField),
+        ProtobufSourcesGeneratorTarget.register_plugin_field(SkipBufField),
+    ]

src/python/pants/backend/codegen/protobuf/lint/buf/subsystem.py

@@ -0,0 +1,37 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from pants.core.util_rules.external_tool import TemplatedExternalTool
+from pants.engine.platform import Platform
+from pants.option.option_types import ArgsListOption, SkipOption
+
+
+class BufSubsystem(TemplatedExternalTool):
+    options_scope = "buf-lint"
+    name = "Buf"
+    help = "A linter for Protocol Buffers (https://github.com/bufbuild/buf)."
+
+    default_version = "v1.0.0"
+    default_known_versions = [
+        "v1.0.0|linux_arm64 |c4b095268fe0fc8de2ad76c7b4677ccd75f25623d5b1f971082a6b7f43ff1eb0|13378006",
+        "v1.0.0|linux_x86_64|5f0ff97576cde9e43ec86959046169f18ec5bcc08e31d82dcc948d057212f7bf|14511545",
+        "v1.0.0|macos_arm64 |e922c277487d941c4b056cac6c1b4c6e5004e8f3dda65ae2d72d8b10da193297|15147463",
+        "v1.0.0|macos_x86_64|8963e1ab7685aac59b8805cc7d752b06a572b1c747a6342a9e73b94ccdf89ddb|15187858",
+    ]
+    default_url_template = (
+        "https://github.com/bufbuild/buf/releases/download/{version}/buf-{platform}.tar.gz"
+    )
+    default_url_platform_mapping = {
+        "macos_arm64": "Darwin-arm64",
+        "macos_x86_64": "Darwin-x86_64",
+        "linux_arm64": "Linux-aarch64",
+        "linux_x86_64": "Linux-x86_64",
+    }
+
+    skip = SkipOption("lint")
+    args = ArgsListOption(example="--error-format json")
+
+    def generate_exe(self, plat: Platform) -> str:
+        return "./buf/bin/buf"