Add intrinsic to download a file (#6660)

* Allow temporary http server to be started

* Disable HTTPS support in grpc

We're finding it hard to link both grpcio (which statically links
openssl) and reqwest (which dynamically links openssl).

We plan to switch to Tower soon, so just disabling HTTPS support at the
grpcio layer for now, which I don't think anyone is relying on.

* Add intrinsic to download a file

src/python/pants/engine/fs.py

@@ -129,6 +129,11 @@ class DirectoryToMaterialize(datatype([('path', text_type), ('directory_digest',
   """A request to materialize the contents of a directory digest at the provided path."""
   pass
 
+
+class UrlToFetch(datatype([('url', text_type), ('digest', Digest)])):
+  pass
+
+
 FilesContent = Collection.of(FileContent)
 
 
@@ -153,4 +158,5 @@ def create_fs_rules():
     RootRule(Digest),
     RootRule(MergedDirectories),
     RootRule(PathGlobs),
+    RootRule(UrlToFetch),
   ]

src/python/pants/engine/native.py

@@ -197,6 +197,7 @@
                             TypeConstraint,
                             TypeConstraint,
                             TypeConstraint,
+                            TypeConstraint,
                             TypeId,
                             TypeId,
                             Buffer,
@@ -823,7 +824,8 @@ def new_scheduler(self,
                     constraint_link,
                     constraint_process_request,
                     constraint_process_result,
-                    constraint_generator):
+                    constraint_generator,
+                    constraint_url_to_fetch):
     """Create and return an ExternContext and native Scheduler."""
 
     def func(constraint):
@@ -856,6 +858,7 @@ def tc(constraint):
         tc(constraint_process_request),
         tc(constraint_process_result),
         tc(constraint_generator),
+        tc(constraint_url_to_fetch),
         # Types.
         TypeId(self.context.to_id(text_type)),
         TypeId(self.context.to_id(binary_type)),

src/python/pants/engine/scheduler.py

@@ -16,7 +16,8 @@
 from pants.base.project_tree import Dir, File, Link
 from pants.build_graph.address import Address
 from pants.engine.fs import (Digest, DirectoryToMaterialize, FileContent, FilesContent,
-                             MergedDirectories, Path, PathGlobs, PathGlobsAndRoot, Snapshot)
+                             MergedDirectories, Path, PathGlobs, PathGlobsAndRoot, Snapshot,
+                             UrlToFetch)
 from pants.engine.isolated_process import ExecuteProcessRequest, FallibleExecuteProcessResult
 from pants.engine.native import Function, TypeConstraint, TypeId
 from pants.engine.nodes import Return, State, Throw
@@ -141,8 +142,10 @@ def __init__(
       constraint_process_request=constraint_for(ExecuteProcessRequest),
       constraint_process_result=constraint_for(FallibleExecuteProcessResult),
       constraint_generator=constraint_for(GeneratorType),
+      constraint_url_to_fetch=constraint_for(UrlToFetch),
     )
 
+
     # If configured, visualize the rule graph before asserting that it is valid.
     if self.visualize_to_dir() is not None:
       rule_graph_name = 'rule_graph.dot'

src/python/pants/util/contextutil.py

@@ -10,19 +10,28 @@
 import signal
 import sys
 import tempfile
+import threading
 import time
 import uuid
 import zipfile
 from builtins import object, open
 from contextlib import closing, contextmanager
 
 from colors import green
-from future.utils import PY3, string_types
+from future.utils import PY2, PY3, string_types
 
 from pants.util.dirutil import safe_delete
 from pants.util.tarutil import TarFile
 
 
+if PY2:
+  from Queue import Queue
+  from SocketServer import TCPServer
+else:
+  from queue import Queue
+  from socketserver import TCPServer
+
+
 class InvalidZipPath(ValueError):
   """Indicates a bad zip file path."""
 
@@ -393,3 +402,23 @@ def with_overwritten_file_content(file_path):
   finally:
     with open(file_path, 'w') as f:
       f.write(file_original_content)
+
+
+@contextmanager
+def http_server(handler_class):
+  def serve(port_queue, shutdown_queue):
+    httpd = TCPServer(("", 0), handler_class)
+    httpd.timeout = 0.1
+    port_queue.put(httpd.server_address[1])
+    while shutdown_queue.empty():
+      httpd.handle_request()
+
+  port_queue = Queue()
+  shutdown_queue = Queue()
+  t = threading.Thread(target=lambda: serve(port_queue, shutdown_queue))
+  t.start()
+
+  yield port_queue.get(block=True)
+
+  shutdown_queue.put(True)
+  t.join()