Add support for auxiliary builds and use it for dataflow builds (#503)…

…. (#582)

* Add AuxiliaryBuild class for extra builds like dataflow (#503).

* Get rid of the AuxiliaryBuild class, use RegularBuild instead.

* fix a test in untrusted_runner

* Revert the change in mock_unpack_build.

* Refactor setup_trunk_build and re-use it + other review comments.

* update the tests, revert some unnecessary changes

* address review comments, use self.env_prefix

src/protos/heartbeat_pb2.py

@@ -32,8 +32,8 @@
   name='heartbeat.proto',
   package='',
   syntax='proto2',
-  serialized_options=None,
-  serialized_pb=_b('\n\x0fheartbeat.proto\"\x12\n\x10HeartbeatRequest\"\x13\n\x11HeartbeatResponse2:\n\tHeartbeat\x12-\n\x04\x42\x65\x61t\x12\x11.HeartbeatRequest\x1a\x12.HeartbeatResponse')
+  serialized_options=_b('Z#clusterfuzz/protos/untrusted_runner'),
+  serialized_pb=_b('\n\x0fheartbeat.proto\"\x12\n\x10HeartbeatRequest\"\x13\n\x11HeartbeatResponse2:\n\tHeartbeat\x12-\n\x04\x42\x65\x61t\x12\x11.HeartbeatRequest\x1a\x12.HeartbeatResponseB%Z#clusterfuzz/protos/untrusted_runner')
 )
 
 
@@ -105,6 +105,7 @@
 _sym_db.RegisterMessage(HeartbeatResponse)
 
 
+DESCRIPTOR._options = None
 
 _HEARTBEAT = _descriptor.ServiceDescriptor(
   name='Heartbeat',

src/protos/heartbeat_pb2_grpc.py

@@ -13,12 +13,9 @@
 # limitations under the License.
 
 # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-from __future__ import absolute_import
-
-from builtins import object
 import grpc
 
-from . import heartbeat_pb2 as heartbeat__pb2
+import heartbeat_pb2 as heartbeat__pb2
 
 
 class HeartbeatStub(object):

src/protos/untrusted_runner.proto

@@ -49,8 +49,8 @@ message SetupRegularBuildRequest {
   optional string base_build_dir = 1;
   optional int64 revision = 2;
   optional string build_url = 3;
-  optional string build_dir_name = 4;
-  map<string, double> target_weights = 5;
+  map<string, double> target_weights = 4;
+  optional string build_prefix = 5;
 }
 
 message SetupSymbolizedBuildRequest {

src/protos/untrusted_runner_pb2_grpc.py

@@ -11,13 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import absolute_import
 
 # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-from builtins import object
 import grpc
 
-from . import untrusted_runner_pb2 as untrusted__runner__pb2
+import untrusted_runner_pb2 as untrusted__runner__pb2
 
 
 class UntrustedRunnerStub(object):

src/python/bot/tasks/fuzz_task.py

@@ -1270,7 +1270,10 @@ def execute_task(fuzzer_name, job_type):
   # Set up a custom or regular build based on revision. By default, fuzzing
   # is done on trunk build (using revision=None). Otherwise, a job definition
   # can provide a revision to use via |APP_REVISION|.
-  build_manager.setup_build(revision=environment.get_value('APP_REVISION'))
+  if build_manager.setup_build(environment.get_value('APP_REVISION')):
+    # Some fuzzing jobs may use auxiliary builds, such as DFSan instrumented
+    # builds accompanying libFuzzer builds to enable DFT-based fuzzing.
+    build_manager.setup_trunk_build(['DATAFLOW_BUILD_BUCKET_PATH'], 'DATAFLOW')
 
   # Check if we have an application path. If not, our build failed
   # to setup correctly.

src/python/bot/untrusted_runner/build_setup.py

@@ -33,23 +33,23 @@ def _build_response(result):
 
 
 def setup_regular_build(request):
-  """Setup a regular build."""
+  """Set up a regular build."""
   build = build_manager.RegularBuild(request.base_build_dir, request.revision,
-                                     request.build_url, request.build_dir_name,
-                                     request.target_weights)
+                                     request.build_url, request.target_weights,
+                                     request.build_prefix)
   return _build_response(build.setup())
 
 
 def setup_symbolized_build(request):
-  """Setup a symbolized build."""
+  """Set up a symbolized build."""
   build = build_manager.SymbolizedBuild(
       request.base_build_dir, request.revision, request.release_build_url,
       request.debug_build_url)
   return _build_response(build.setup())
 
 
 def setup_production_build(request):
-  """Setup a production build."""
+  """Set up a production build."""
   build = build_manager.ProductionBuild(request.base_build_dir, request.version,
                                         request.build_url, request.build_type)
   return _build_response(build.setup())

src/python/bot/untrusted_runner/build_setup_host.py

@@ -67,7 +67,7 @@ def setup(self):
         base_build_dir=self.base_build_dir,
         revision=self.revision,
         build_url=self.build_url,
-        build_dir_name=self.build_dir_name)
+        build_prefix=self.build_prefix)
     if self.target_weights:
       request.target_weights.update(self.target_weights)