Make setup_py tasks provide 'python_dists' product.

This enables subsequent tasks to consume the artifacts produced
by the setup_py task.

src/python/pants/backend/python/tasks/setup_py.py

@@ -298,6 +298,10 @@ def is_resources_target(target):
   def has_provides(cls, target):
     return cls.is_python_target(target) and target.provides
 
+  @classmethod
+  def product_types(cls):
+    return ['python_dists']
+
   class DependencyCalculator(ExportedTargetDependencyCalculator):
     """Calculates reduced dependencies for exported python targets."""
 
@@ -609,7 +613,7 @@ def create(target):
     for target in targets:
       create(target)
 
-    executed = {}  # Collected and returned for tests, processed target -> sdist|setup_dir.
+    python_dists = self.context.products.get('python_dists')
     for target in reversed(sort_targets(created.keys())):
       setup_dir = created.get(target)
       if setup_dir:
@@ -621,10 +625,10 @@ def create(target):
           self.context.log.info('Writing {}'.format(sdist_path))
           shutil.move(setup_runner.sdist(), sdist_path)
           safe_rmtree(setup_dir)
-          executed[target] = sdist_path
+          python_dists.add(target, os.path.dirname(sdist_path)).append(os.path.basename(sdist_path))
         else:
           self.context.log.info('Running {} against {}'.format(self._run, setup_dir))
           setup_runner = SetupPyRunner(setup_dir, self._run)
           setup_runner.run()
-          executed[target] = setup_dir
-    return executed
+          python_dists.add(target, os.path.dirname(setup_dir)).append(os.path.basename(setup_dir))
+    return python_dists  # returned for tests

src/python/pants/backend/python/tasks2/setup_py.py

@@ -293,6 +293,10 @@ def is_resources_target(target):
   def has_provides(cls, target):
     return cls.is_python_target(target) and target.provides
 
+  @classmethod
+  def product_types(cls):
+    return ['python_dists']
+
   class DependencyCalculator(ExportedTargetDependencyCalculator):
     """Calculates reduced dependencies for exported python targets."""
 
@@ -592,7 +596,7 @@ def create(target):
     for target in targets:
       create(target)
 
-    executed = {}  # Collected and returned for tests, processed target -> sdist|setup_dir.
+    python_dists = self.context.products.get('python_dists')
     for target in reversed(sort_targets(created.keys())):
       setup_dir = created.get(target)
       if setup_dir:
@@ -604,10 +608,10 @@ def create(target):
           self.context.log.info('Writing {}'.format(sdist_path))
           shutil.move(setup_runner.sdist(), sdist_path)
           safe_rmtree(setup_dir)
-          executed[target] = sdist_path
+          python_dists.add(target, os.path.dirname(sdist_path)).append(os.path.basename(sdist_path))
         else:
           self.context.log.info('Running {} against {}'.format(self._run, setup_dir))
           setup_runner = SetupPyRunner(setup_dir, self._run)
           setup_runner.run()
-          executed[target] = setup_dir
-    return executed
+          python_dists.add(target, os.path.dirname(setup_dir)).append(os.path.basename(setup_dir))
+    return python_dists  # returned for tests

tests/python/pants_test/backend/python/tasks/test_setup_py.py

@@ -95,10 +95,10 @@ def test_execution_reduced_dependencies_1(self):
     dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
     target_map = self.create_dependencies(dep_map)
     with self.run_execute(target_map['foo'], recursive=False) as created:
-      self.assertEqual([target_map['foo']], created.keys())
+      self.assertEqual([target_map['foo']], created.by_target.keys())
     with self.run_execute(target_map['foo'], recursive=True) as created:
       self.assertEqual({target_map['baz'], target_map['bar'], target_map['foo']},
-                       set(created.keys()))
+                       set(created.by_target.keys()))
 
   def test_reduced_dependencies_2(self):
     # foo --> baz
@@ -161,10 +161,10 @@ def test_binary_target_injected_into_reduced_dependencies(self):
     self.assertEqual(entry_points, {'foo_binary': 'foo.bin:foo'})
 
     with self.run_execute(foo, recursive=False) as created:
-      self.assertEqual([foo], created.keys())
+      self.assertEqual([foo], created.by_target.keys())
 
     with self.run_execute(foo, recursive=True) as created:
-      self.assertEqual([foo], created.keys())
+      self.assertEqual([foo], created.by_target.keys())
 
   def test_binary_target_injected_into_reduced_dependencies_with_provider(self):
     bar_bin_dep = self.create_python_library(
@@ -207,10 +207,10 @@ def test_binary_target_injected_into_reduced_dependencies_with_provider(self):
     self.assertEqual(entry_points, {'bar_binary': 'bar.bin:bar'})
 
     with self.run_execute(bar, recursive=False) as created:
-      self.assertEqual([bar], created.keys())
+      self.assertEqual([bar], created.by_target.keys())
 
     with self.run_execute(bar, recursive=True) as created:
-      self.assertEqual({bar_bin_dep, bar}, set(created.keys()))
+      self.assertEqual({bar_bin_dep, bar}, set(created.by_target.keys()))
 
   def test_pants_contrib_case(self):
     def create_requirement_lib(name):
@@ -392,9 +392,10 @@ def test_resources(self):
     conway = self.target('src/python/monster:conway')
 
     with self.run_execute(conway) as created:
-      self.assertEqual([conway], created.keys())
+      self.assertEqual([conway], created.by_target.keys())
+      sdist = os.path.join(created[conway].items()[0][0], created[conway].items()[0][1][0])
 
-      with self.extracted_sdist(sdist=created[conway],
+      with self.extracted_sdist(sdist=sdist,
                                 expected_prefix='monstrous.moonshine-0.0.0',
                                 collect_suffixes=('.py', '.res')) as (py_files, path):
         self.assertEqual({path('setup.py'),
@@ -434,14 +435,15 @@ def test_symlinks_issues_2815(self):
     conway = self.target('src/python/monster:conway')
 
     with self.run_execute(conway) as created:
-      self.assertEqual([conway], created.keys())
+      self.assertEqual([conway], created.by_target.keys())
 
       # Now that we've created the sdist tarball, delete the symlink destination to ensure the
       # unpacked sdist can't get away with unpacking a symlink that happens to have local
       # resolution.
       os.unlink(res)
 
-      with self.extracted_sdist(sdist=created[conway],
+      sdist = os.path.join(created[conway].items()[0][0], created[conway].items()[0][1][0])
+      with self.extracted_sdist(sdist=sdist,
                                 expected_prefix='monstrous.moonshine-0.0.0',
                                 collect_suffixes=('.py', '.res')) as (py_files, path):
         res_link_path = path('src/monster/group.res')
@@ -481,7 +483,7 @@ def test_exported_antlr(self):
     # Remove this once proper Subsystem dependency chains are re-established.
     init_subsystem(JVM)
     with self.run_execute(target) as created:
-      self.assertEqual([target], created.keys())
+      self.assertEqual([target], created.by_target.keys())
 
   def test_exported_thrift(self):
     self.create_file(relpath='src/thrift/exported/exported.thrift', contents=dedent("""
@@ -494,7 +496,7 @@ def test_exported_thrift(self):
                               sources=['exported.thrift'],
                               provides=PythonArtifact(name='test.exported', version='0.0.0'))
     with self.run_execute(target) as created:
-      self.assertEqual([target], created.keys())
+      self.assertEqual([target], created.by_target.keys())
 
   def test_exported_thrift_issues_2005(self):
     # Issue #2005 highlighted the fact the PythonThriftBuilder was building both a given
@@ -528,9 +530,10 @@ def test_exported_thrift_issues_2005(self):
                                                        version='0.0.0'))
 
     with self.run_execute(target2, recursive=True) as created:
-      self.assertEqual({target2, target1}, set(created.keys()))
+      self.assertEqual({target2, target1}, set(created.by_target.keys()))
+      sdist1 = os.path.join(created[target1].items()[0][0], created[target1].items()[0][1][0])
 
-      with self.extracted_sdist(sdist=created[target1],
+      with self.extracted_sdist(sdist=sdist1,
                                 expected_prefix='test.exported-0.0.0') as (py_files, path):
         self.assertEqual({path('setup.py'),
                           path('src/__init__.py'),
@@ -541,7 +544,9 @@ def test_exported_thrift_issues_2005(self):
 
         self.assertFalse(os.path.exists(path('src/test.exported.egg-info/requires.txt')))
 
-      with self.extracted_sdist(sdist=created[target2],
+      sdist2 = os.path.join(created[target2].items()[0][0], created[target2].items()[0][1][0])
+
+      with self.extracted_sdist(sdist=sdist2,
                                 expected_prefix='test.exported_dependee-0.0.0') as (py_files, path):
         self.assertEqual({path('setup.py'),
                           path('src/__init__.py'),
@@ -603,7 +608,7 @@ def test_prep_command_case(self):
       ]
     )
     with self.run_execute(pants) as created:
-      self.assertEqual([pants], created.keys())
+      self.assertEqual([pants], created.by_target.keys())
 
 
 def test_detect_namespace_packages():

tests/python/pants_test/backend/python/tasks2/test_setup_py.py

@@ -91,10 +91,10 @@ def test_execution_reduced_dependencies_1(self):
     dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
     target_map = self.create_dependencies(dep_map)
     with self.run_execute(target_map['foo'], recursive=False) as created:
-      self.assertEqual([target_map['foo']], created.keys())
+      self.assertEqual([target_map['foo']], created.by_target.keys())
     with self.run_execute(target_map['foo'], recursive=True) as created:
       self.assertEqual({target_map['baz'], target_map['bar'], target_map['foo']},
-                       set(created.keys()))
+                       set(created.by_target.keys()))
 
   def test_reduced_dependencies_2(self):
     # foo --> baz
@@ -157,10 +157,10 @@ def test_binary_target_injected_into_reduced_dependencies(self):
     self.assertEqual(entry_points, {'foo_binary': 'foo.bin:foo'})
 
     with self.run_execute(foo, recursive=False) as created:
-      self.assertEqual([foo], created.keys())
+      self.assertEqual([foo], created.by_target.keys())
 
     with self.run_execute(foo, recursive=True) as created:
-      self.assertEqual([foo], created.keys())
+      self.assertEqual([foo], created.by_target.keys())
 
   def test_binary_target_injected_into_reduced_dependencies_with_provider(self):
     bar_bin_dep = self.create_python_library(
@@ -203,10 +203,10 @@ def test_binary_target_injected_into_reduced_dependencies_with_provider(self):
     self.assertEqual(entry_points, {'bar_binary': 'bar.bin:bar'})
 
     with self.run_execute(bar, recursive=False) as created:
-      self.assertEqual([bar], created.keys())
+      self.assertEqual([bar], created.by_target.keys())
 
     with self.run_execute(bar, recursive=True) as created:
-      self.assertEqual({bar_bin_dep, bar}, set(created.keys()))
+      self.assertEqual({bar_bin_dep, bar}, set(created.by_target.keys()))
 
   def test_pants_contrib_case(self):
     def create_requirement_lib(name):
@@ -388,9 +388,10 @@ def test_resources(self):
     conway = self.target('src/python/monster:conway')
 
     with self.run_execute(conway) as created:
-      self.assertEqual([conway], created.keys())
+      self.assertEqual([conway], created.by_target.keys())
+      sdist = os.path.join(created[conway].items()[0][0], created[conway].items()[0][1][0])
 
-      with self.extracted_sdist(sdist=created[conway],
+      with self.extracted_sdist(sdist=sdist,
                                 expected_prefix='monstrous.moonshine-0.0.0',
                                 collect_suffixes=('.py', '.res')) as (py_files, path):
         self.assertEqual({path('setup.py'),
@@ -430,14 +431,15 @@ def test_symlinks_issues_2815(self):
     conway = self.target('src/python/monster:conway')
 
     with self.run_execute(conway) as created:
-      self.assertEqual([conway], created.keys())
+      self.assertEqual([conway], created.by_target.keys())
 
       # Now that we've created the sdist tarball, delete the symlink destination to ensure the
       # unpacked sdist can't get away with unpacking a symlink that happens to have local
       # resolution.
       os.unlink(res)
+      sdist = os.path.join(created[conway].items()[0][0], created[conway].items()[0][1][0])
 
-      with self.extracted_sdist(sdist=created[conway],
+      with self.extracted_sdist(sdist=sdist,
                                 expected_prefix='monstrous.moonshine-0.0.0',
                                 collect_suffixes=('.py', '.res')) as (py_files, path):
         res_link_path = path('src/monster/group.res')
@@ -500,7 +502,7 @@ def test_prep_command_case(self):
       ]
     )
     with self.run_execute(pants) as created:
-      self.assertEqual([pants], created.keys())
+      self.assertEqual([pants], created.by_target.keys())
 
 
 def test_detect_namespace_packages():