[gn] Consolidate exec_script calls to speed up generation

Calling out to python from GN to process gypi files is relatively
expensive with a 20-45ms fixed overhead for setup/teardown regardless
of what the script does. This makes runtime/vm/BUILD.gn take 1-1.5s
(per toolchain) to run as the template for libraries expands
out to 25 calls to gypi_to_gn.py, even though the actual time spent
processing the gypi files is negligible.

This replaces those repeated calls to gypi_to_gn.py with a call to a
custom script that process all of the gypi files needed for runtime/vm
and places the results into a single scope which can then be read from
in the template and replaces a few other scattered calls to
gypi_to_gn.py with a smaller number of calls that process multiple
gypi files and place the results into a single scope.

The end result is processing all of dart's GN files in a fuchsia build
takes ~250ms instead of >3 seconds.

R=zra@google.com

Review URL: https://codereview.chromium.org/2472813002 .

Committed: ad86d6e

runtime/bin/BUILD.gn

@@ -2,6 +2,8 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+import("gypi_contents.gni")
+
 declare_args() {
   # Whether to fall back to built-in root certificates when they cannot be
   # verified at the operating system level.
@@ -18,12 +20,6 @@ declare_args() {
   dart_use_tcmalloc = false
 }
 
-resources_sources_gypi =
-    exec_script("../../tools/gypi_to_gn.py",
-                [ rebase_path("vmservice/vmservice_sources.gypi") ],
-                "scope",
-                [ "vmservice/vmservice_sources.gypi" ])
-
 # Generate a resources.cc file for the service isolate without Observatory.
 action("gen_resources_cc") {
   visibility = [ ":*" ]  # Only targets in this file can see this.
@@ -34,7 +30,7 @@ action("gen_resources_cc") {
 
   # The path below is hard coded for the Mojo and Flutter trees. When moving
   # the Dart runtime to gn, this path might need to be updated.
-  sources = rebase_path(resources_sources_gypi.sources, "", "../bin/vmservice/")
+  sources = rebase_path(resources_sources_gypi, "", "../bin/vmservice/")
   outputs = [
     "$target_gen_dir/resources_gen.cc",
   ]
@@ -86,15 +82,10 @@ template("gen_library_src_path") {
   }
 }
 
-builtin_sources_gypi = exec_script("../../tools/gypi_to_gn.py",
-                                   [ rebase_path("builtin_sources.gypi") ],
-                                   "scope",
-                                   [ "builtin_sources.gypi" ])
-
 gen_library_src_path("generate_builtin_cc_file") {
   name = "_builtin"
   kind = "source"
-  sources = builtin_sources_gypi.sources
+  sources = builtin_sources_gypi
   output = "$target_gen_dir/builtin_gen.cc"
 }
 
@@ -113,15 +104,10 @@ gen_library_src_path("generate_io_cc_file") {
   output = "$target_gen_dir/io_gen.cc"
 }
 
-io_sources_gypi = exec_script("../../tools/gypi_to_gn.py",
-                              [ rebase_path("io_sources.gypi") ],
-                              "scope",
-                              [ "io_sources.gypi" ])
-
 gen_library_src_path("generate_io_patch_cc_file") {
   name = "io"
   kind = "patch"
-  sources = io_sources_gypi.sources
+  sources = io_sources_gypi
   output = "$target_gen_dir/io_patch_gen.cc"
 }
 
@@ -253,12 +239,6 @@ config("libdart_builtin_config") {
   }
 }
 
-builtin_impl_sources_gypi =
-    exec_script("../../tools/gypi_to_gn.py",
-                [ rebase_path("builtin_impl_sources.gypi") ],
-                "scope",
-                [ "builtin_impl_sources.gypi" ])
-
 static_library("libdart_builtin") {
   configs += [
     "..:dart_config",
@@ -293,14 +273,9 @@ static_library("libdart_builtin") {
               "log_linux.cc",
               "log_macos.cc",
               "log_win.cc",
-            ] + builtin_impl_sources_gypi.sources
+            ] + builtin_impl_sources_gypi
 }
 
-io_impl_sources_gypi = exec_script("../../tools/gypi_to_gn.py",
-                                   [ rebase_path("io_impl_sources.gypi") ],
-                                   "scope",
-                                   [ "io_impl_sources.gypi" ])
-
 executable("gen_snapshot") {
   configs += [
     "..:dart_config",
@@ -386,7 +361,7 @@ source_set("gen_snapshot_dart_io") {
 
   defines = [ "DART_IO_SECURE_SOCKET_DISABLED" ]
 
-  sources = io_impl_sources_gypi.sources + builtin_impl_sources_gypi.sources
+  sources = io_impl_sources_gypi + builtin_impl_sources_gypi
   sources += [
     "io_natives.cc",
     "io_natives.h",
@@ -452,7 +427,7 @@ template("dart_io") {
       ]
     }
 
-    sources = io_impl_sources_gypi.sources + builtin_impl_sources_gypi.sources
+    sources = io_impl_sources_gypi + builtin_impl_sources_gypi
     sources += [
                  "builtin_natives.cc",
                  "io_natives.cc",
@@ -810,30 +785,12 @@ executable("run_vm_tests") {
 
   # The VM sources are already included in libdart, so we just want to add in
   # the tests here.
-  vm_tests_list = exec_script("../../tools/gypi_to_gn.py",
-                              [
-                                rebase_path("../vm/vm_sources.gypi"),
-                                "--keep_only=_test.cc",
-                                "--keep_only=_test.h",
-                              ],
-                              "scope",
-                              [ "../vm/vm_sources.gypi" ])
-  vm_tests = rebase_path(vm_tests_list.sources, ".", "../vm")
-
-  builtin_impl_tests_list =
-      exec_script("../../tools/gypi_to_gn.py",
-                  [
-                    rebase_path("builtin_impl_sources.gypi"),
-                    "--keep_only=_test.cc",
-                    "--keep_only=_test.h",
-                  ],
-                  "scope",
-                  [ "builtin_impl_sources.gypi" ])
+  vm_tests = rebase_path(vm_tests_list, ".", "../vm")
 
   sources = [
               "builtin_nolib.cc",
               "run_vm_tests.cc",
-            ] + builtin_impl_tests_list.sources + vm_tests
+            ] + builtin_impl_tests_list + vm_tests
 
   if (!is_win) {
     ldflags = [ "-rdynamic" ]

runtime/bin/gypi_contents.gni

@@ -0,0 +1,43 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+_gypi_files = [
+  "builtin_sources.gypi",
+  "builtin_impl_sources.gypi",
+  "vmservice/vmservice_sources.gypi",
+  "io_sources.gypi",
+  "io_impl_sources.gypi",
+]
+
+_gypi_contents = exec_script("../../tools/gypi_to_gn.py",
+                             rebase_path(_gypi_files) + [ "--prefix" ],
+                             "scope",
+                             _gypi_files)
+
+resources_sources_gypi = _gypi_contents.vmservice_sources_sources
+
+builtin_sources_gypi = _gypi_contents.builtin_sources_sources
+builtin_impl_sources_gypi = _gypi_contents.builtin_impl_sources_sources
+
+io_sources_gypi = _gypi_contents.io_sources_sources
+io_impl_sources_gypi = _gypi_contents.io_impl_sources_sources
+
+_test_gypi_files = [
+  "../vm/vm_sources.gypi",
+  "builtin_impl_sources.gypi",
+]
+
+_test_only_gypi_contents = exec_script("../../tools/gypi_to_gn.py",
+                                       rebase_path(_test_gypi_files) + [
+                                             "--keep_only=_test.cc",
+                                             "--keep_only=_test.h",
+                                             "--prefix",
+                                           ],
+                                       "scope",
+                                       _test_gypi_files)
+
+# The VM sources are already included in libdart, so we just want to add in
+# the tests here.
+vm_tests_list = _test_only_gypi_contents.vm_sources_sources
+builtin_impl_tests_list = _test_only_gypi_contents.builtin_impl_sources_sources