Replace cloc with count-loc, using much faster SCC program (#10740)

* Improve `cloc` to use the much faster SCC program

# Rust tests and lints will be skipped. Delete if not intended.
[ci skip-rust]

# Building wheels and fs_util will be skipped. Delete if not intended.
[ci skip-build-wheels]

* Remove stray string from iterating

[ci skip-rust]
[ci skip-build-wheels]

* Review feedback

[ci skip-rust]
[ci skip-build-wheels]

* Move `--args` to `scc` scope

# Rust tests and lints will be skipped. Delete if not intended.
[ci skip-rust]

# Building wheels and fs_util will be skipped. Delete if not intended.
[ci skip-build-wheels]

src/python/pants/backend/project_info/count_loc.py

@@ -0,0 +1,105 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from typing import Tuple
+
+from pants.core.util_rules.external_tool import (
+    DownloadedExternalTool,
+    ExternalTool,
+    ExternalToolRequest,
+)
+from pants.engine.console import Console
+from pants.engine.fs import Digest, MergeDigests, SourcesSnapshot
+from pants.engine.goal import Goal, GoalSubsystem
+from pants.engine.platform import Platform
+from pants.engine.process import Process, ProcessResult
+from pants.engine.rules import Get, collect_rules, goal_rule
+from pants.option.custom_types import shell_str
+from pants.util.enums import match
+from pants.util.logging import LogLevel
+from pants.util.strutil import pluralize
+
+
+class SuccinctCodeCounter(ExternalTool):
+    """The Succinct Code Counter, aka `scc` (https://github.com/boyter/scc)."""
+
+    options_scope = "scc"
+    default_version = "2.12.0"
+    default_known_versions = [
+        "2.12.0|darwin|70b7002cd1e4541cb37b7b9cbc0eeedd13ceacb49628e82ab46332bb2e65a5a6|1842530",
+        "2.12.0|linux|8eca3e98fe8a78d417d3779a51724515ac4459760d3ec256295f80954a0da044|1753059",
+    ]
+
+    @classmethod
+    def register_options(cls, register) -> None:
+        super().register_options(register)
+        register(
+            "--args",
+            type=list,
+            member_type=shell_str,
+            passthrough=True,
+            help=(
+                'Arguments to pass directly to SCC, e.g. `--count-loc-args="--no-cocomo"`. Refer '
+                "to https://github.com/boyter/scc."
+            ),
+        )
+
+    @property
+    def args(self) -> Tuple[str, ...]:
+        return tuple(self.options.args)
+
+    def generate_url(self, plat: Platform) -> str:
+        plat_str = match(plat, {Platform.darwin: "apple-darwin", Platform.linux: "unknown-linux"})
+        return (
+            f"https://github.com/boyter/scc/releases/download/v{self.version}/scc-{self.version}-"
+            f"x86_64-{plat_str}.zip"
+        )
+
+    def generate_exe(self, _: Platform) -> str:
+        return "./scc"
+
+
+class CountLinesOfCodeSubsystem(GoalSubsystem):
+    """Count lines of code using `scc` (Succinct Code Counter, https://github.com/boyter/scc)."""
+
+    name = "count-loc"
+
+
+class CountLinesOfCode(Goal):
+    subsystem_cls = CountLinesOfCodeSubsystem
+
+
+@goal_rule
+async def count_loc(
+    console: Console,
+    succinct_code_counter: SuccinctCodeCounter,
+    sources_snapshot: SourcesSnapshot,
+) -> CountLinesOfCode:
+    if not sources_snapshot.snapshot.files:
+        return CountLinesOfCode(exit_code=0)
+
+    scc_program = await Get(
+        DownloadedExternalTool,
+        ExternalToolRequest,
+        succinct_code_counter.get_request(Platform.current),
+    )
+    input_digest = await Get(
+        Digest, MergeDigests((scc_program.digest, sources_snapshot.snapshot.digest))
+    )
+    result = await Get(
+        ProcessResult,
+        Process(
+            argv=(scc_program.exe, *succinct_code_counter.args),
+            input_digest=input_digest,
+            description=(
+                f"Count lines of code for {pluralize(len(sources_snapshot.snapshot.files), 'file')}"
+            ),
+            level=LogLevel.DEBUG,
+        ),
+    )
+    console.print_stdout(result.stdout.decode())
+    return CountLinesOfCode(exit_code=0)
+
+
+def rules():
+    return collect_rules()

src/python/pants/backend/project_info/cloc_test.py → src/python/pants/backend/project_info/count_loc_test.py

@@ -3,8 +3,8 @@
 
 import pytest
 
-from pants.backend.project_info import cloc
-from pants.backend.project_info.cloc import CountLinesOfCode
+from pants.backend.project_info import count_loc
+from pants.backend.project_info.count_loc import CountLinesOfCode
 from pants.backend.python.target_types import PythonLibrary
 from pants.core.util_rules import external_tool
 from pants.engine.target import Sources, Target
@@ -23,7 +23,8 @@ class ElixirTarget(Target):
 @pytest.fixture
 def rule_runner() -> RuleRunner:
     return RuleRunner(
-        rules=[*cloc.rules(), *external_tool.rules()], target_types=[PythonLibrary, ElixirTarget]
+        rules=[*count_loc.rules(), *external_tool.rules()],
+        target_types=[PythonLibrary, ElixirTarget],
     )
 
 
@@ -34,19 +35,19 @@ def assert_counts(
         (
             line
             for line in stdout.splitlines()
-            if len(line.split()) == 5 and line.split()[0] == lang
+            if len(line.split()) in (6, 7) and line.split()[0] == lang
         ),
         None,
     )
-    assert summary_line is not None, f"Found no output line for {lang}"
+    assert summary_line is not None, f"Found no output line for {lang} given stdout:\n {stdout}"
     fields = summary_line.split()
     assert num_files == int(fields[1])
-    assert blank == int(fields[2])
-    assert comment == int(fields[3])
-    assert code == int(fields[4])
+    assert blank == int(fields[3])
+    assert comment == int(fields[4])
+    assert code == int(fields[5])
 
 
-def test_cloc(rule_runner: RuleRunner) -> None:
+def test_count_loc(rule_runner: RuleRunner) -> None:
     py_dir = "src/py/foo"
     rule_runner.create_file(
         f"{py_dir}/foo.py", '# A comment.\n\nprint("some code")\n# Another comment.'
@@ -67,33 +68,18 @@ def test_cloc(rule_runner: RuleRunner) -> None:
     assert_counts(result.stdout, "Elixir", comment=1, code=1)
 
 
-def test_ignored(rule_runner: RuleRunner) -> None:
-    py_dir = "src/py/foo"
-    rule_runner.create_file(f"{py_dir}/foo.py", "print('some code')")
-    rule_runner.create_file(f"{py_dir}/empty.py", "")
-    rule_runner.add_to_build_file(py_dir, "python_library()")
-
-    result = rule_runner.run_goal_rule(CountLinesOfCode, args=[py_dir, "--cloc-ignored"])
-    assert result.exit_code == 0
-    assert "Ignored the following files:" in result.stderr
-    assert "empty.py: zero sized file" in result.stderr
-
-
-def test_filesystem_specs_with_owners(rule_runner: RuleRunner) -> None:
-    """Even if a file belongs to a target which has multiple sources, we should only run over the
-    specified file."""
-    py_dir = "src/py/foo"
-    rule_runner.create_file(f"{py_dir}/foo.py", "print('some code')")
-    rule_runner.create_file(f"{py_dir}/bar.py", "print('some code')\nprint('more code')")
-    rule_runner.add_to_build_file(py_dir, "python_library()")
-    result = rule_runner.run_goal_rule(CountLinesOfCode, args=[f"{py_dir}/foo.py"])
+def test_passthrough_args(rule_runner: RuleRunner) -> None:
+    rule_runner.create_file("foo.py", "print('hello world!')\n")
+    rule_runner.add_to_build_file("", "python_library(name='foo')")
+    result = rule_runner.run_goal_rule(CountLinesOfCode, args=["//:foo", "--", "--no-cocomo"])
     assert result.exit_code == 0
-    assert_counts(result.stdout, "Python", num_files=1, code=1)
+    assert_counts(result.stdout, "Python", code=1)
+    assert "Estimated Cost to Develop" not in result.stdout
 
 
-def test_filesystem_specs_without_owners(rule_runner: RuleRunner) -> None:
-    """Unlike most goals, cloc works on any readable file in the build root, regardless of whether
-    it's declared in a BUILD file."""
+def test_files_without_owners(rule_runner: RuleRunner) -> None:
+    """cloc works on any readable file in the build root, regardless of whether it's declared in a
+    BUILD file."""
     rule_runner.create_file("test/foo.ex", 'IO.puts("im a free thinker!")')
     rule_runner.create_file("test/foo.hs", 'main = putStrLn "Whats Pants, precious?"')
     result = rule_runner.run_goal_rule(CountLinesOfCode, args=["test/foo.*"])

src/python/pants/backend/project_info/register.py

@@ -4,7 +4,7 @@
 """Information on your project, such as listing the targets in your project."""
 
 from pants.backend.project_info import (
-    cloc,
+    count_loc,
     dependees,
     dependencies,
     filedeps,
@@ -17,7 +17,7 @@
 
 def rules():
     return [
-        *cloc.rules(),
+        *count_loc.rules(),
         *dependees.rules(),
         *dependencies.rules(),
         *filedeps.rules(),