Reapply "[build] Don't list Dart sources up front when creating appli…

…cation snapshots." Fix `application_snapshot`'s depfile to track the sources of the application instead of the compiler. Split compiling the compiler into a separate GN target with its own depfile. Bug: flutter/flutter#81074 Change-Id: I0fb23ada40a6241ee3dde7f6cfebdd121b9a4224 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/197020 Reviewed-by: Alexander Aprelev <aam@google.com> Commit-Queue: Ryan Macnak <rmacnak@google.com>
dart-lang · Apr 27, 2021 · 605b211 · 605b211
1 parent 69b6cee
commit 605b211
Show file tree

Hide file tree

Showing 9 changed files with 63 additions and 147 deletions.
diff --git a/pkg/BUILD.gn b/pkg/BUILD.gn
@@ -2,61 +2,6 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
-import("../utils/create_timestamp.gni")
-
-template("make_third_party_pkg_files_stamp") {
-  assert(defined(invoker.id), "Must define the stamp file id")
-  path = rebase_path("../third_party/pkg")
-  if (defined(invoker.path)) {
-    path = invoker.path
-  }
-  id = invoker.id
-  create_timestamp_file(target_name) {
-    if (defined(invoker.pattern)) {
-      pattern = invoker.pattern
-    }
-    path = path
-    output = "$target_gen_dir/third_party_pkg_files_$id.stamp"
-  }
-}
-
-make_third_party_pkg_files_stamp("make_third_party_pkg_files_0_stamp") {
-  path = rebase_path(".")
-  id = "0"
-}
-
-make_third_party_pkg_files_stamp("make_third_party_pkg_files_1_stamp") {
-  pattern = "[a-k].*"
-  id = "1"
-}
-
-make_third_party_pkg_files_stamp("make_third_party_pkg_files_2_stamp") {
-  pattern = "[l-r].*"
-  id = "2"
-}
-
-make_third_party_pkg_files_stamp("make_third_party_pkg_files_3_stamp") {
-  pattern = "[s-z].*"
-  id = "3"
-}
-
-action("pkg_files_stamp") {
-  deps = [
-    ":make_third_party_pkg_files_0_stamp",
-    ":make_third_party_pkg_files_1_stamp",
-    ":make_third_party_pkg_files_2_stamp",
-    ":make_third_party_pkg_files_3_stamp",
-  ]
-
-  stamp0_outputs = get_target_outputs(":make_third_party_pkg_files_0_stamp")
-  stamp1_outputs = get_target_outputs(":make_third_party_pkg_files_1_stamp")
-  stamp2_outputs = get_target_outputs(":make_third_party_pkg_files_2_stamp")
-  stamp3_outputs = get_target_outputs(":make_third_party_pkg_files_3_stamp")
-
-  inputs = stamp0_outputs + stamp1_outputs + stamp2_outputs + stamp3_outputs
-
-  outputs = [ "$root_gen_dir/pkg_files.stamp" ]
-
-  script = "../tools/create_timestamp_file.py"
-  args = [ rebase_path("$root_gen_dir/pkg_files.stamp") ]
+# TODO(flutter): Remove use.
+group("pkg_files_stamp") {
 }
diff --git a/utils/analysis_server/BUILD.gn b/utils/analysis_server/BUILD.gn
@@ -4,13 +4,6 @@
 
 import("../application_snapshot.gni")
 
-analysis_server_files = exec_script("../../tools/list_dart_files.py",
-                                    [
-                                      "absolute",
-                                      rebase_path("../../pkg/analysis_server"),
-                                    ],
-                                    "list lines")
-
 application_snapshot("analysis_server") {
   main_dart = "../../pkg/analysis_server/bin/server.dart"
   training_args = [
@@ -19,5 +12,4 @@ application_snapshot("analysis_server") {
     # "--sdk=" + rebase_path("../../sdk/"),
     # "--train-using=" + rebase_path("../../pkg/analyzer_cli")
   ]
-  inputs = analysis_server_files
 }
diff --git a/utils/application_snapshot.gni b/utils/application_snapshot.gni
@@ -85,12 +85,15 @@ template("_application_snapshot") {
     deps = extra_deps + [
              "$_dart_root/runtime/vm:kernel_platform_files($host_toolchain)",
              "$_dart_root/runtime/vm:vm_platform",
+             "$_dart_root/utils/gen_kernel:bootstrap_gen_kernel",
            ]
-    gen_kernel_script = "$_dart_root/pkg/vm/bin/gen_kernel.dart"
+    gen_kernel_kernel =
+        get_label_info("$_dart_root/utils/gen_kernel:bootstrap_gen_kernel",
+                       "target_gen_dir") + "/bootstrap_gen_kernel.dill"
     platform_dill = "$root_out_dir/vm_platform_strong.dill"
 
     inputs = extra_inputs + [
-               gen_kernel_script,
+               gen_kernel_kernel,
                platform_dill,
                main_dart,
                dot_packages,
@@ -99,26 +102,22 @@ template("_application_snapshot") {
     outputs = [ output ]
 
     depfile = "$output.d"
-    abs_depfile = rebase_path(depfile)
-    rebased_output = rebase_path(output, root_build_dir)
 
     vm_args = [
-      "--depfile=$abs_depfile",
-      "--depfile_output_filename=$rebased_output",
-
       # Ensure gen_kernel.dart will use this SDK hash when consuming/producing kernel.
       "-Dsdk_hash=$sdk_hash",
     ]
 
-    script = gen_kernel_script
+    script = gen_kernel_kernel
 
     args = [
       "--packages=" + rebase_path(dot_packages),
       "--platform=" + rebase_path(platform_dill),
       "--no-aot",
       "--no-embed-sources",
       "--no-link-platform",
-      "--output=" + rebase_path(output),
+      "--output=" + rebase_path(output, root_build_dir),
+      "--depfile=" + rebase_path(depfile),
 
       # Ensure the compiled appliation (e.g. kernel-service, frontend-server,
       # ...) will use this SDK hash when consuming/producing kernel.

diff --git a/utils/bazel/BUILD.gn b/utils/bazel/BUILD.gn
@@ -7,5 +7,4 @@ import("../application_snapshot.gni")
 application_snapshot("kernel_worker") {
   main_dart = "kernel_worker.dart"
   training_args = [ "--help" ]
-  deps = [ "../../pkg:pkg_files_stamp" ]
 }
diff --git a/utils/dartanalyzer/BUILD.gn b/utils/dartanalyzer/BUILD.gn
@@ -4,6 +4,7 @@
 
 import("../../build/dart/dart_action.gni")
 import("../application_snapshot.gni")
+import("../create_timestamp.gni")
 
 group("dartanalyzer") {
   deps = [
@@ -12,13 +13,6 @@ group("dartanalyzer") {
   ]
 }
 
-analyzer_files = exec_script("../../tools/list_dart_files.py",
-                             [
-                               "absolute",
-                               rebase_path("../../pkg/analyzer"),
-                             ],
-                             "list lines")
-
 application_snapshot("generate_dartanalyzer_snapshot") {
   main_dart = "../../pkg/analyzer_cli/bin/analyzer.dart"
   training_args = [
@@ -31,21 +25,21 @@ application_snapshot("generate_dartanalyzer_snapshot") {
   name = "dartanalyzer"
 }
 
-sdk_lib_files = exec_script("../../tools/list_dart_files.py",
-                            [
-                              "absolute",
-                              rebase_path("../../sdk/lib"),
-                            ],
-                            "list lines")
+sdk_root = "../../sdk"
+
+create_timestamp_file("sdk_lib_files") {
+  path = rebase_path("$sdk_root/lib")
+  output = "$target_gen_dir/sdk_lib_files.stamp"
+}
 
 dart_action("generate_summary_strong") {
   deps = [
+    ":sdk_lib_files",
     "../../sdk:copy_libraries",
     "../../sdk:write_version_file",
   ]
   script = "../../pkg/analyzer/tool/summary/build_sdk_summaries.dart"
   packages = "../../.packages"
-  inputs = sdk_lib_files + analyzer_files
   output = "$root_gen_dir/strong.sum"
   outputs = [ output ]
   vm_args = [ "-Dsdk_hash=$sdk_hash" ]

diff --git a/utils/dartdev/BUILD.gn b/utils/dartdev/BUILD.gn
@@ -4,13 +4,6 @@
 
 import("../application_snapshot.gni")
 
-dartdev_files = exec_script("../../tools/list_dart_files.py",
-                            [
-                              "absolute",
-                              rebase_path("../../pkg/dartdev"),
-                            ],
-                            "list lines")
-
 group("dartdev") {
   public_deps = [
     ":copy_dartdev_kernel",
@@ -30,7 +23,6 @@ application_snapshot("generate_dartdev_kernel") {
   main_dart = "../../pkg/dartdev/bin/dartdev.dart"
   training_args = []
   deps = [ "../dds:dds" ]
-  inputs = dartdev_files
   output = "$root_gen_dir/dartdev.dill"
 }
 
@@ -45,6 +37,5 @@ application_snapshot("generate_dartdev_snapshot") {
   main_dart = "../../pkg/dartdev/bin/dartdev.dart"
   training_args = [ "--help" ]
   deps = [ "../dds:dds" ]
-  inputs = dartdev_files
   output = "$root_gen_dir/dartdev.dart.snapshot"
 }
diff --git a/utils/dartdevc/BUILD.gn b/utils/dartdevc/BUILD.gn
@@ -40,27 +40,6 @@ application_snapshot("dartdevc") {
 
 sdk_root = "../../sdk"
 
-sdk_lib_files = exec_script("../../tools/list_dart_files.py",
-                            [
-                              "absolute",
-                              rebase_path("$sdk_root/lib"),
-                            ],
-                            "list lines")
-
-compiler_files = exec_script("../../tools/list_dart_files.py",
-                             [
-                               "absolute",
-                               rebase_path("../../pkg/compiler"),
-                             ],
-                             "list lines")
-
-dev_compiler_files = exec_script("../../tools/list_dart_files.py",
-                                 [
-                                   "absolute",
-                                   rebase_path("../../pkg/dev_compiler"),
-                                 ],
-                                 "list lines")
-
 template("dart2js_compile") {
   assert(defined(invoker.main), "Must specify the main file")
   main = invoker.main
@@ -70,12 +49,12 @@ template("dart2js_compile") {
   abs_output = rebase_path(out)
 
   prebuilt_dart_action(target_name) {
-    deps = [ "../compiler:compile_dart2js_platform" ]
+    deps = invoker.deps + [ "../compiler:compile_dart2js_platform" ]
 
-    inputs = sdk_lib_files + compiler_files + dev_compiler_files + [
-               "$root_out_dir/dart2js_platform.dill",
-               "$root_out_dir/dart2js_outline.dill",
-             ]
+    inputs = [
+      "$root_out_dir/dart2js_platform.dill",
+      "$root_out_dir/dart2js_outline.dill",
+    ]
     outputs = [ out ]
 
     script = "../../pkg/compiler/lib/src/dart2js.dart"
@@ -97,6 +76,7 @@ template("dart2js_compile") {
 dart2js_compile("stack_trace_mapper") {
   main = rebase_path("../../pkg/dev_compiler/web/stack_trace_mapper.dart")
   out = "$root_out_dir/dev_compiler/build/web/dart_stack_trace_mapper.js"
+  deps = [ ":dartdevc_files_stamp" ]
 }
 
 # Builds everything needed to run dartdevc tests using test.dart.
@@ -180,16 +160,11 @@ template("dartdevc_kernel_compile") {
     deps = [
       ":dartdevc_files_stamp",
       platform_dep,
-
-      # TODO(sigmund): depend only on the compiler and the actual files in the
-      # package
-      "../../pkg:pkg_files_stamp",
     ]
 
     inputs = [
       sdk_outline,
       "$target_gen_dir/dartdevc_files.stamp",
-      "$root_gen_dir/pkg_files.stamp",
     ]
 
     outputs = [
@@ -327,17 +302,9 @@ template("dartdevc_sdk_js") {
   }
 
   prebuilt_dart_action(target_name) {
-    deps = [
-      ":dartdevc_files_stamp",
-      "../../pkg:pkg_files_stamp",
-      platform_dep,
-    ]
+    deps = [ platform_dep ]
 
-    inputs = [
-      "$target_gen_dir/dartdevc_files.stamp",
-      "$root_gen_dir/pkg_files.stamp",
-      platform_input,
-    ]
+    inputs = [ platform_input ]
 
     outputs = [
       "$js_gen_dir/amd/dart_sdk.js",

diff --git a/utils/dds/BUILD.gn b/utils/dds/BUILD.gn
@@ -4,13 +4,6 @@
 
 import("../application_snapshot.gni")
 
-dds_files = exec_script("../../tools/list_dart_files.py",
-                        [
-                          "absolute",
-                          rebase_path("../../pkg/dds"),
-                        ],
-                        "list lines")
-
 group("dds") {
   public_deps = [ ":copy_dds_snapshot" ]
 }
@@ -25,6 +18,5 @@ copy("copy_dds_snapshot") {
 application_snapshot("generate_dds_snapshot") {
   main_dart = "../../pkg/dds/bin/dds.dart"
   training_args = []
-  inputs = dds_files
   output = "$root_gen_dir/dds.dart.snapshot"
 }
diff --git a/utils/gen_kernel/BUILD.gn b/utils/gen_kernel/BUILD.gn
@@ -2,6 +2,43 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+import("../../build/dart/dart_action.gni")
+import("../../sdk_args.gni")
+
+_dart_root = get_path_info("../..", "abspath")
+
+prebuilt_dart_action("bootstrap_gen_kernel") {
+  deps = [
+    "$_dart_root/runtime/vm:kernel_platform_files($host_toolchain)",
+    "$_dart_root/runtime/vm:vm_platform",
+  ]
+  gen_kernel_script = "$_dart_root/pkg/vm/bin/gen_kernel.dart"
+  platform_dill = "$root_out_dir/vm_platform_strong.dill"
+  dot_packages = rebase_path("$_dart_root/.packages")
+
+  inputs = [
+    gen_kernel_script,
+    platform_dill,
+    dot_packages,
+  ]
+  output = "$target_gen_dir/bootstrap_gen_kernel.dill"
+  outputs = [ output ]
+
+  depfile = "$output.d"
+  vm_args = [
+    "--snapshot-kind=kernel",
+    "--snapshot=" + rebase_path(output, root_build_dir),
+    "--depfile=" + rebase_path(depfile),
+
+    # Ensure gen_kernel.dart will use this SDK hash when consuming/producing kernel.
+    "-Dsdk_hash=$sdk_hash",
+
+    "--packages=" + rebase_path(dot_packages),
+  ]
+  script = gen_kernel_script
+  args = []
+}
+
 import("../application_snapshot.gni")
 
 application_snapshot("gen_kernel") {