Cherry pick PR #476: Add args for launching blackboxtests for specific test set

.github/actions/on_host_test/action.yaml

@@ -71,6 +71,10 @@ runs:
         fi
         if [[ "${{matrix.shard}}" == 'integration' ]]; then
           xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args}
+        elif [[ "${{matrix.shard}}" == 'blackbox' ]]; then
+          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set blackbox
+        elif [[ "${{matrix.shard}}" == 'wpt' ]]; then
+          xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/black_box_tests/black_box_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --test_set wpt
         elif [[ "${{matrix.shard}}" == 'evergreen' ]]; then
           xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 $GITHUB_WORKSPACE/cobalt/evergreen_tests/evergreen_tests.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} ${loader_args} --no-can_mount_tmpfs
         else

.github/config/evergreen-x64.json

@@ -2,7 +2,7 @@
   "docker_service": "build-linux-evergreen",
   "on_host_test": true,
   "bootloader": "linux-x64x11",
-  "on_host_test_shards": ["0", "1", "2", "3", "integration", "evergreen"],
+  "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt", "evergreen"],
   "platforms": [
     "evergreen-x64",
     "evergreen-x64-sbversion-15",

.github/config/linux-clang-3-9.json

@@ -1,7 +1,7 @@
 {
   "docker_service": "build-linux-clang-3-9",
   "on_host_test": true,
-  "on_host_test_shards": ["0", "1", "2", "3", "integration"],
+  "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt"],
   "platforms": [
     "linux-x64x11-clang-3-9"
   ],

.github/config/linux.json

@@ -1,7 +1,7 @@
 {
   "docker_service": "build-linux",
   "on_host_test": true,
-  "on_host_test_shards": ["0", "1", "2", "3", "integration"],
+  "on_host_test_shards": ["0", "1", "2", "3", "blackbox", "wpt"],
   "platforms": [
     "linux-x64x11",
     "linux-x64x11-egl",

cobalt/black_box_tests/black_box_tests.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # Copyright 2017 The Cobalt Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -92,11 +93,14 @@
     'text_encoding_test',
     'wasm_basic_test',
     'web_debugger',
-    'web_platform_tests',
     'web_worker_test',
     'worker_csp_test',
     'worker_load_test',
 ]
+# These are very different and require a custom config + proxy
+_WPT_TESTS = [
+    'web_platform_tests',
+]
 # These tests can only be run on platforms whose app launcher can send deep
 # links.
 _TESTS_NEEDING_DEEP_LINK = [
@@ -114,7 +118,7 @@
 # Platform configuration and device information parameters.
 _launcher_params = None
 # Binding address used to create the test server.
-_binding_address = None
+_server_binding_address = None
 # Port used to create the web platform test http server.
 _wpt_http_port = None
 
@@ -132,6 +136,7 @@ def __init__(self, *args, **kwargs):
   @classmethod
   def setUpClass(cls):
     super(BlackBoxTestCase, cls).setUpClass()
+    logging.info('\n\n\n%s\n\n', '=' * 40)
     logging.info('Running %s', cls.__name__)
 
   @classmethod
@@ -151,13 +156,13 @@ def CreateCobaltRunner(self, url=None, target_params=None, **kwargs):
         **kwargs)
 
   def GetBindingAddress(self):
-    return _binding_address
+    return _server_binding_address
 
   def GetWptHttpPort(self):
     return _wpt_http_port
 
 
-def LoadTests(launcher_params):
+def LoadTests(launcher_params, test_set):
   launcher = abstract_launcher.LauncherFactory(
       launcher_params.platform,
       _LAUNCH_TARGET,
@@ -170,13 +175,19 @@ def LoadTests(launcher_params):
       loader_config=launcher_params.loader_config,
       loader_out_directory=launcher_params.loader_out_directory)
 
-  test_targets = _TESTS_NO_SIGNAL
+  test_targets = []
+
+  if test_set in ['all', 'blackbox']:
+    test_targets = _TESTS_NO_SIGNAL
 
-  if launcher.SupportsSuspendResume():
-    test_targets += _TESTS_NEEDING_SYSTEM_SIGNAL
+    if launcher.SupportsSuspendResume():
+      test_targets += _TESTS_NEEDING_SYSTEM_SIGNAL
 
-  if launcher.SupportsDeepLink():
-    test_targets += _TESTS_NEEDING_DEEP_LINK
+    if launcher.SupportsDeepLink():
+      test_targets += _TESTS_NEEDING_DEEP_LINK
+
+  if test_set in ['all', 'wpt']:
+    test_targets += _WPT_TESTS
 
   test_suite = unittest.TestSuite()
   for test in test_targets:
@@ -213,50 +224,49 @@ def LoadEvergreenEndToEndTests(launcher_params):
 class BlackBoxTests(object):
   """Helper class to run all black box tests and return results."""
 
-  def __init__(self,
-               server_binding_address,
-               proxy_address=None,
-               proxy_port=None,
-               test_name=None,
-               wpt_http_port=None,
-               device_ips=None,
-               device_id=None):
+  def __init__(self, args):
+
+    self.args = args
+
+    #TODO(b/137905502): These globals should be refactored
     # Setup global variables used by test cases.
     global _launcher_params
     _launcher_params = command_line.CreateLauncherParams()
     # Keep other modules from seeing these args.
     sys.argv = sys.argv[:1]
-    global _binding_address
-    _binding_address = server_binding_address
+    global _server_binding_address
+    _server_binding_address = args.server_binding_address
+
     # Port used to create the web platform test http server. If not specified,
     # a random free port is used.
-    if wpt_http_port is None:
-      wpt_http_port = str(self.GetUnusedPort([server_binding_address]))
     global _wpt_http_port
-    _wpt_http_port = wpt_http_port
+    _wpt_http_port = args.wpt_http_port or str(
+        self.GetUnusedPort([_server_binding_address]))
+
+    # Proxy is only needed for WPT
+    self.use_proxy = args.test_set in ['all', 'wpt']
+
     # TODO: Remove generation of --dev_servers_listen_ip once executable will
     # be able to bind correctly with incomplete support of IPv6
-    if device_id and IsValidIpAddress(device_id):
+    if args.device_id and IsValidIpAddress(args.device_id):
       _launcher_params.target_params.append(
-          f'--dev_servers_listen_ip={device_id}')
-    elif IsValidIpAddress(server_binding_address):
+          f'--dev_servers_listen_ip={args.device_id}')
+    elif IsValidIpAddress(_server_binding_address):
       _launcher_params.target_params.append(
-          f'--dev_servers_listen_ip={server_binding_address}')
+          f'--dev_servers_listen_ip={_server_binding_address}')
     _launcher_params.target_params.append(
-        f'--web-platform-test-server=http://web-platform.test:{wpt_http_port}')
+        f'--web-platform-test-server=http://web-platform.test:{_wpt_http_port}')
 
     # Port used to create the proxy server. If not specified, a random free
     # port is used.
-    if proxy_port is None:
-      proxy_port = str(self.GetUnusedPort([server_binding_address]))
-    if proxy_address is None:
-      proxy_address = server_binding_address
-    _launcher_params.target_params.append(
-        f'--proxy={proxy_address}:{proxy_port}')
+    if self.use_proxy:
+      self.proxy_port = args.proxy_port or str(
+          self.GetUnusedPort([_server_binding_address]))
+      proxy_address = args.proxy_address or _server_binding_address
+      _launcher_params.target_params.append(
+          f'--proxy={proxy_address}:{self.proxy_port}')
 
-    self.proxy_port = proxy_port
-    self.test_name = test_name
-    self.device_ips = device_ips
+    self.device_ips = args.device_ips
 
     # Test domains used in web platform tests to be resolved to the server
     # binding address.
@@ -265,43 +275,40 @@ def __init__(self,
         'www2.web-platform.test', 'xn--n8j6ds53lwwkrqhv28a.web-platform.test',
         'xn--lve-6lad.web-platform.test'
     ]
-    self.host_resolve_map = {host: server_binding_address for host in hosts}
+    self.host_resolve_map = {host: _server_binding_address for host in hosts}
 
   def Run(self):
-    if self.proxy_port == '-1':
+    if self.use_proxy and self.proxy_port == '-1':
       return 1
 
     run_cobalt_tests = True
     run_evergreen_tests = False
     launch_config = f'{_launcher_params.platform}/{_launcher_params.config}'
+    # TODO(b/135549281): Configuring this in Python is superfluous, the on/off
+    # flags can be in Github Actions code
     if launch_config in _DISABLED_BLACKBOXTEST_CONFIGS:
       run_cobalt_tests = False
       logging.warning(
           'Cobalt blackbox tests disabled for platform:%s config:%s',
           _launcher_params.platform, _launcher_params.config)
 
     if launch_config in _EVERGREEN_COMPATIBLE_CONFIGS:
-      run_evergreen_tests = True
+      run_evergreen_tests = self.args.test_set in ['all', 'evergreen']
 
     if not (run_cobalt_tests or run_evergreen_tests):
       return 0
 
-    logging.info('Using proxy port: %s', self.proxy_port)
-
-    with ProxyServer(
-        port=self.proxy_port,
-        host_resolve_map=self.host_resolve_map,
-        client_ips=self.device_ips):
-      if self.test_name:
+    def LoadAndRunTests():
+      if self.args.test_name:
         suite = unittest.TestLoader().loadTestsFromName(_TEST_DIR_PATH +
-                                                        self.test_name)
+                                                        self.args.test_name)
         return_code = not unittest.TextTestRunner(
             verbosity=2, stream=sys.stdout).run(suite).wasSuccessful()
         return return_code
       else:
         cobalt_tests_return_code = 0
         if run_cobalt_tests:
-          suite = LoadTests(_launcher_params)
+          suite = LoadTests(_launcher_params, self.args.test_set)
           # Using verbosity=2 to log individual test function names and results.
           cobalt_tests_return_code = not unittest.TextTestRunner(
               verbosity=2, stream=sys.stdout).run(suite).wasSuccessful()
@@ -314,6 +321,16 @@ def Run(self):
 
         return cobalt_tests_return_code or evergreen_tests_return_code
 
+    if self.use_proxy:
+      logging.info('Using proxy port: %s', self.proxy_port)
+      with ProxyServer(
+          port=self.proxy_port,
+          host_resolve_map=self.host_resolve_map,
+          client_ips=self.args.device_ips):
+        return LoadAndRunTests()
+    else:
+      return LoadAndRunTests()
+
   def GetUnusedPort(self, addresses):
     """Find a free port on the list of addresses by pinging with sockets."""
 
@@ -410,14 +427,15 @@ def main():
       nargs='*',
       help=('IPs of test devices that will be allowed to connect. If not '
             'specified, all IPs will be allowed to connect.'))
+  parser.add_argument(
+      '--test_set',
+      choices=['all', 'wpt', 'blackbox', 'evergreen'],
+      default='all')
   args, _ = parser.parse_known_args()
 
   log_level.InitializeLogging(args)
 
-  test_object = BlackBoxTests(args.server_binding_address, args.proxy_address,
-                              args.proxy_port, args.test_name,
-                              args.wpt_http_port, args.device_ips,
-                              args.device_id)
+  test_object = BlackBoxTests(args)
   sys.exit(test_object.Run())
 
 

tools/create_archive.py

@@ -282,15 +282,15 @@ def _CreateLinuxTarCmd(source_path, intermediate_tar_path, patterns,
       if glob.glob(os.path.join(source_path, pattern))
   ]
   files_to_tar = ' '.join(contents)
-  return (f'tar -{mode}vf {intermediate_tar_path} --format=posix '
+  return (f'tar -{mode}f {intermediate_tar_path} --format=posix '
           f'{excludes} {files_to_tar}')
 
 
 def _CreateUntarCommand(intermediate_tar_path):
   if _IsWindows():
     return f'"{_7Z_PATH}" x -bsp1 {intermediate_tar_path}'
   else:
-    return f'tar -xvf {intermediate_tar_path}'
+    return f'tar -xf {intermediate_tar_path}'
 
 
 def _CreateZipCommand(intermediate_tar_path, dest_path, is_parallel=False):