Subsystems for the ragel and cloc binaries (#5517)

Also renames protobuf/subsystem -> protobuf/subsystems, for consistency.

contrib/buildrefactor/src/python/pants/contrib/buildrefactor/buildozer_binary.py

@@ -17,6 +17,8 @@
 
 
 class BuildozerBinary(NativeTool):
+  # Note: Not in scope 'buildozer' because that's the name of the singleton task
+  # that runs buildozer.
   options_scope = 'buildozer-binary'
   name = 'buildozer'
   default_version = '0.6.0.dce8b3c287652cbcaf43c8dd076b3f48c92ab44c'

src/python/pants/backend/codegen/protobuf/java/BUILD

@@ -4,7 +4,7 @@
 python_library(
   dependencies = [
     '3rdparty/python/twitter/commons:twitter.common.collections',
-    'src/python/pants/backend/codegen/protobuf/subsystem',
+    'src/python/pants/backend/codegen/protobuf/subsystems',
     'src/python/pants/backend/jvm/targets:java',
     'src/python/pants/backend/jvm/targets:jvm',
     'src/python/pants/backend/jvm/tasks:jar_import_products',

src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py

@@ -12,7 +12,7 @@
 from twitter.common.collections import OrderedSet
 
 from pants.backend.codegen.protobuf.java.java_protobuf_library import JavaProtobufLibrary
-from pants.backend.codegen.protobuf.subsystem.protoc import Protoc
+from pants.backend.codegen.protobuf.subsystems.protoc import Protoc
 from pants.backend.jvm.targets.jar_library import JarLibrary
 from pants.backend.jvm.targets.java_library import JavaLibrary
 from pants.backend.jvm.tasks.jar_import_products import JarImportProducts

src/python/pants/backend/codegen/ragel/java/BUILD

@@ -4,9 +4,9 @@
 python_library(
   dependencies = [
     'src/python/pants/backend/jvm/targets:java',
+    'src/python/pants/backend/codegen/ragel/subsystems',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:exceptions',
-    'src/python/pants/binaries:binary_util',
     'src/python/pants/goal:task_registrar',
     'src/python/pants/task',
     'src/python/pants/util:contextutil',

src/python/pants/backend/codegen/ragel/java/ragel_gen.py

@@ -9,10 +9,10 @@
 import re
 
 from pants.backend.codegen.ragel.java.java_ragel_library import JavaRagelLibrary
+from pants.backend.codegen.ragel.subsystems.ragel import Ragel
 from pants.backend.jvm.targets.java_library import JavaLibrary
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
-from pants.binaries.binary_util import BinaryUtil
 from pants.task.simple_codegen_task import SimpleCodegenTask
 from pants.util.dirutil import safe_mkdir_for
 from pants.util.memo import memoized_property
@@ -22,18 +22,20 @@
 class RagelGen(SimpleCodegenTask):
   @classmethod
   def subsystem_dependencies(cls):
-    return super(RagelGen, cls).subsystem_dependencies() + (BinaryUtil.Factory,)
+    return super(RagelGen, cls).subsystem_dependencies() + (Ragel.scoped(cls),)
 
   @classmethod
   def register_options(cls, register):
     super(RagelGen, cls).register_options(register)
     register('--supportdir', default='bin/ragel', advanced=True,
+             removal_version='1.7.0.dev0', removal_hint='No longer in use.',
              help='The path to find the ragel binary.  Used as part of the path to lookup the'
                   'tool with --pants-support-baseurls and --pants_bootstrapdir.')
 
     # We take the cautious approach here and assume a version bump will always correspond to
     # changes in ragel codegen products.
     register('--version', default='6.9', advanced=True, fingerprint=True,
+             removal_version='1.7.0.dev0', removal_hint='Use --version in scope ragel.',
              help='The version of ragel to use.  Used as part of the path to lookup the'
                   'tool with --pants-support-baseurls and --pants-bootstrapdir')
 
@@ -43,10 +45,7 @@ def __init__(self, *args, **kwargs):
 
   @memoized_property
   def ragel_binary(self):
-    binary_util = BinaryUtil.Factory.create()
-    return binary_util.select_binary(self.get_options().supportdir,
-                                     self.get_options().version,
-                                     'ragel')
+    return Ragel.scoped_instance(self).select(context=self.context)
 
   def synthetic_target_type(self, target):
     return JavaLibrary

src/python/pants/backend/codegen/ragel/subsystems/BUILD

@@ -0,0 +1,8 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies = [
+    'src/python/pants/binaries:binary_util'
+  ],
+)

src/python/pants/backend/codegen/ragel/subsystems/ragel.py

@@ -0,0 +1,16 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.binaries.binary_tool import NativeTool
+
+
+class Ragel(NativeTool):
+  options_scope = 'ragel'
+  default_version = '6.9'
+
+  replaces_scope = 'gen.ragel'
+  replaces_name = 'version'

src/python/pants/backend/graph_info/subsystems/BUILD

@@ -0,0 +1,8 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies = [
+    'src/python/pants/binaries:binary_util',
+  ],
+)

src/python/pants/backend/graph_info/subsystems/cloc_binary.py

@@ -0,0 +1,18 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.binaries.binary_tool import Script
+
+
+class ClocBinary(Script):
+  # Note: Not in scope 'cloc' because that's the name of the singleton task that runs cloc.
+  options_scope = 'cloc-binary'
+  name = 'cloc'
+  default_version = '1.66'
+
+  replaces_scope = 'cloc'
+  replaces_name = 'version'

src/python/pants/backend/graph_info/tasks/BUILD

@@ -3,13 +3,13 @@
 
 python_library(
   dependencies = [
+    'src/python/pants/backend/graph_info/subsystems',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:cmd_line_spec_parser',
     'src/python/pants/base:deprecated',
     'src/python/pants/base:exceptions',
     'src/python/pants/base:specs',
     'src/python/pants/base:workunit',
-    'src/python/pants/binaries:binary_util',
     'src/python/pants/build_graph',
     'src/python/pants/goal',
     'src/python/pants/task',

src/python/pants/backend/graph_info/tasks/cloc.py

@@ -7,10 +7,10 @@
 
 import os
 
+from pants.backend.graph_info.subsystems.cloc_binary import ClocBinary
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnitLabel
-from pants.binaries.binary_util import BinaryUtil
 from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult
 from pants.task.console_task import ConsoleTask
 from pants.util.contextutil import temporary_dir
@@ -22,21 +22,22 @@ class CountLinesOfCode(ConsoleTask):
 
   @classmethod
   def subsystem_dependencies(cls):
-    return super(CountLinesOfCode, cls).subsystem_dependencies() + (BinaryUtil.Factory,)
+    return super(CountLinesOfCode, cls).subsystem_dependencies() + (ClocBinary,)
 
   @classmethod
   def register_options(cls, register):
     super(CountLinesOfCode, cls).register_options(register)
-    register('--version', advanced=True, fingerprint=True, default='1.66', help='Version of cloc.')
+    register('--version', advanced=True, fingerprint=True, default='1.66',
+             removal_version='1.7.0.dev0', removal_hint='Use --version in scope cloc-binary',
+             help='Version of cloc.')
     register('--transitive', type=bool, fingerprint=True, default=True,
              help='Operate on the transitive dependencies of the specified targets.  '
                   'Unset to operate only on the specified targets.')
     register('--ignored', type=bool, fingerprint=True,
              help='Show information about files ignored by cloc.')
 
   def _get_cloc_script(self):
-    binary_util = BinaryUtil.Factory.create()
-    return binary_util.select_script('scripts/cloc', self.get_options().version, 'cloc')
+    return ClocBinary.global_instance().select(self.context)
 
   def console_output(self, targets):
     if not self.get_options().transitive:

tests/python/pants_test/backend/codegen/ragel/java/test_ragel_gen.py

@@ -68,7 +68,6 @@ def test_ragel_gen(self):
                               target_type=JavaRagelLibrary,
                               sources=['atoi.rl'])
     task = self.create_task(self.context(target_roots=[target]))
-
     target_workdir = safe_mkdtemp(dir=self.test_workdir)
     task.execute_codegen(target, target_workdir)
 

tests/python/pants_test/base_test.py

@@ -320,13 +320,13 @@ def context(self, for_task_types=None, for_subsystems=None, options=None,
       scoped_opts = options.setdefault(s, {})
       scoped_opts.update(opts)
 
-    options = create_options_for_optionables(
+    fake_options = create_options_for_optionables(
       optionables, extra_scopes=extra_scopes, options=options, **kwargs)
 
     Subsystem.reset(reset_options=True)
-    Subsystem.set_options(options)
+    Subsystem.set_options(fake_options)
 
-    context = create_context_from_options(options,
+    context = create_context_from_options(fake_options,
                                           target_roots=target_roots,
                                           build_graph=self.build_graph,
                                           build_file_parser=self.build_file_parser,