[web] add CanvasKit to CIPD; make it a DEPS dependency; add a manual …

…roller script (#28056)

add CanvasKit to CIPD; make it a DEPS dependency; add a manual roller script

DEPS

@@ -29,6 +29,10 @@ vars = {
   'ocmock_git': 'https://github.com/erikdoe/ocmock.git',
   'skia_revision': '73339ada9d13ed284179989029b94fa878015bb7',
 
+  # WARNING: DO NOT EDIT canvaskit_cipd_instance MANUALLY
+  # See `lib/web_ui/README.md` for how to roll CanvasKit to a new version.
+  'canvaskit_cipd_instance': 'srPdrAoUIWOde-KMPE5S8koi64mejhae7NzvfR_7m3gC',
+
   # When updating the Dart revision, ensure that all entries that are
   # dependencies of Dart are also updated to match the entries in the
   # Dart SDK's DEPS file for that revision of Dart. The DEPS file for
@@ -535,6 +539,16 @@ deps = {
      'dep_type': 'cipd',
    },
 
+  'src/third_party/web_dependencies': {
+     'packages': [
+       {
+         'package': 'flutter/web/canvaskit_bundle',
+         'version': Var('canvaskit_cipd_instance')
+       }
+     ],
+     'dep_type': 'cipd',
+   },
+
   'src/third_party/java/openjdk': {
      'packages': [
        {

ci/licenses_golden/tool_signature

@@ -1,2 +1,2 @@
-Signature: 35d96b1f4c55cfa89ee2a17ff46bba47
+Signature: ba62ed10d5e046f90a397663d6c9d5d2
 

lib/web_ui/README.md

@@ -1 +1,29 @@
-To run tests, use `dev/felt test`. See dev/README.md for details.
+# Flutter Web Engine
+
+This directory contains the source code for the Web Engine. The easiest way to
+hack on the Web Engine is using the `felt` tool. See dev/README.md for details.
+
+## Rolling CanvasKit
+
+CanvasKit is versioned separately from Skia and rolled manually. Flutter
+consumes a pre-built CanvasKit provided by the Skia team, currently hosted on
+unpkg.com. When a new version of CanvasKit is available (check
+https://www.npmjs.com/package/canvaskit-wasm or consult the Skia team
+directly), follow these steps to roll to the new version:
+
+- Make sure you have `depot_tools` installed (if you are regularly hacking on
+  the engine code, you probably do).
+- If not already authenticated with CIPD, run `cipd auth-login` and follow
+  instructions (this step requires sufficient privileges; contact
+  #hackers-infra-🌡 on Flutter's Discord server).
+- Edit `dev/canvaskit_lock.yaml` and update the value of `canvaskit_version`
+  to the new version.
+- Run `dart dev/canvaskit_roller.dart` and make sure it completes successfully.
+  The script uploads the new version of CanvasKit to the
+  `flutter/web/canvaskit_bundle` CIPD package, and writes the CIPD package
+  instance ID to the DEPS file.
+- Send a pull request containing the above file changes. If the new version
+  contains breaking changes, the PR must also contain corresponding fixes.
+
+If you have questions, contact the Flutter Web team on Flutter Discord on the
+#hackers-web-🌍 channel.

lib/web_ui/dev/canvaskit_lock.yaml

@@ -0,0 +1,4 @@
+# Specifies the version of CanvasKit to use for Flutter Web apps.
+#
+# See `lib/web_ui/README.md` for how to update this file.
+canvaskit_version: "0.28.1"

lib/web_ui/dev/canvaskit_roller.dart

@@ -0,0 +1,168 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert' show json;
+import 'dart:io';
+
+import 'package:path/path.dart' as pathlib;
+import 'package:yaml/yaml.dart';
+
+import 'environment.dart';
+import 'utils.dart';
+
+/// Rolls CanvasKit to the version specified in `dev/canvaskit_lock.yaml`.
+///
+/// Detailed instructions for how to use this script can be found in
+/// `lib/web_ui/README.md`.
+Future<void> main(List<String> args) async {
+  final String canvaskitVersion = _readCanvaskitVersion();
+  print('Rolling CanvasKit to version $canvaskitVersion');
+
+  final Directory canvaskitDirectory = await Directory.systemTemp.createTemp('canvaskit-roll-$canvaskitVersion-');
+  print('Will use ${canvaskitDirectory.path} as staging directory.');
+
+  final String baseUrl = 'https://unpkg.com/canvaskit-wasm@$canvaskitVersion/bin/';
+  print('Downloading CanvasKit from $baseUrl');
+  final HttpClient client = HttpClient();
+  for (final String assetPath in _canvaskitAssets) {
+    final String assetUrl = '$baseUrl/$assetPath';
+    final File assetFile = File(pathlib.joinAll(<String>[
+      canvaskitDirectory.path,
+      'canvaskit',
+      ...assetPath.split('/'), // so it's compatible with Windows
+    ]));
+    await assetFile.parent.create(recursive: true);
+    final HttpClientRequest request = await client.getUrl(Uri.parse(assetUrl));
+    final HttpClientResponse response = await request.close();
+    final IOSink fileSink = assetFile.openWrite();
+    await response.pipe(fileSink);
+  }
+  client.close();
+
+  final File cipdConfigFile = File(pathlib.join(
+    canvaskitDirectory.path,
+    'cipd.yaml',
+  ));
+  await cipdConfigFile.writeAsString('''
+package: flutter/web/canvaskit_bundle
+description: A build of CanvasKit bundled with Flutter Web apps
+data:
+  - dir: canvaskit
+''');
+
+  print('Uploading to CIPD');
+  await runProcess('cipd', <String>[
+    'create',
+    '--tag=version:$canvaskitVersion',
+    '--pkg-def=cipd.yaml',
+    '--json-output=result.json',
+  ], workingDirectory: canvaskitDirectory.path);
+
+  final Map<String, dynamic> cipdResult = json.decode(File(pathlib.join(
+    canvaskitDirectory.path,
+    'result.json',
+  )).readAsStringSync()) as Map<String, dynamic>;
+  final String cipdInstanceId = cipdResult['result']['instance_id'] as String;
+
+  print('CIPD instance information:');
+  final String cipdInfo = await evalProcess('cipd', <String>[
+    'describe',
+    'flutter/web/canvaskit_bundle',
+    '--version=$cipdInstanceId',
+  ], workingDirectory: canvaskitDirectory.path);
+  print(cipdInfo.trim().split('\n').map((String line) => ' • $line').join('\n'));
+
+  print('Updating DEPS file');
+  await _updateDepsFile(cipdInstanceId);
+  await _updateCanvaskitInitializationCode(canvaskitVersion);
+
+  print('\nATTENTION: the roll process is not complete yet.');
+  print('Last step: for the roll to take effect submit an engine pull request from local git changes.');
+}
+
+const List<String> _canvaskitAssets = <String>[
+  'canvaskit.js',
+  'canvaskit.wasm',
+  'profiling/canvaskit.js',
+  'profiling/canvaskit.wasm',
+];
+
+String _readCanvaskitVersion() {
+  final YamlMap canvaskitLock = loadYaml(File(pathlib.join(
+    environment.webUiDevDir.path,
+    'canvaskit_lock.yaml',
+  )).readAsStringSync()) as YamlMap;
+  return canvaskitLock['canvaskit_version'] as String;
+}
+
+Future<void> _updateDepsFile(String cipdInstanceId) async {
+  final File depsFile = File(pathlib.join(
+    environment.flutterDirectory.path,
+    'DEPS',
+  ));
+
+  final String originalDepsCode = await depsFile.readAsString();
+  final List<String> rewrittenDepsCode = <String>[];
+  const String kCanvasKitDependencyKeyInDeps = '\'canvaskit_cipd_instance\': \'';
+  bool canvaskitDependencyFound = false;
+  for (final String line in originalDepsCode.split('\n')) {
+    if (line.trim().startsWith(kCanvasKitDependencyKeyInDeps)) {
+      canvaskitDependencyFound = true;
+      rewrittenDepsCode.add(
+        "  'canvaskit_cipd_instance': '$cipdInstanceId',",
+      );
+    } else {
+      rewrittenDepsCode.add(line);
+    }
+  }
+
+  if (!canvaskitDependencyFound) {
+    stderr.writeln(
+      'Failed to update the DEPS file.\n'
+      'Could not to locate CanvasKit dependency in the DEPS file. Make sure the '
+      'DEPS file contains a line like this:\n'
+      '\n'
+      '  \'canvaskit_cipd_instance\': \'SOME_VALUE\','
+    );
+    exit(1);
+  }
+
+  await depsFile.writeAsString(rewrittenDepsCode.join('\n'));
+}
+
+Future<void> _updateCanvaskitInitializationCode(String canvaskitVersion) async {
+  const String kCanvasKitVersionKey = 'const String canvaskitVersion';
+  const String kPathToInitializationCode = 'lib/src/engine/canvaskit/initialization.dart';
+  final File initializationFile = File(pathlib.join(
+    environment.webUiRootDir.path,
+    kPathToInitializationCode,
+  ));
+  final String originalInitializationCode = await initializationFile.readAsString();
+
+  final List<String> rewrittenCode = <String>[];
+  bool canvaskitVersionFound = false;
+  for (final String line in originalInitializationCode.split('\n')) {
+    if (line.trim().startsWith(kCanvasKitVersionKey)) {
+      canvaskitVersionFound = true;
+      rewrittenCode.add(
+        "const String canvaskitVersion = '$canvaskitVersion';",
+      );
+    } else {
+      rewrittenCode.add(line);
+    }
+  }
+
+  if (!canvaskitVersionFound) {
+    stderr.writeln(
+      'Failed to update CanvasKit version in $kPathToInitializationCode.\n'
+      'Could not to locate the constant that defines the version. Make sure the '
+      '$kPathToInitializationCode file contains a line like this:\n'
+      '\n'
+      'const String canvaskitVersion = \'VERSION\';'
+    );
+    exit(1);
+  }
+
+  await initializationFile.writeAsString(rewrittenCode.join('\n'));
+}

lib/web_ui/dev/utils.dart

@@ -56,6 +56,27 @@ Future<int> runProcess(
   return manager.wait();
 }
 
+/// Runs the process and returns its standard output as a string.
+///
+/// Standard error output is ignored (use [ProcessManager.evalStderr] for that).
+///
+/// Throws an exception if the process exited with a non-zero exit code.
+Future<String> evalProcess(
+  String executable,
+  List<String> arguments, {
+  String? workingDirectory,
+  Map<String, String> environment = const <String, String>{},
+}) async {
+  final ProcessManager manager = await startProcess(
+    executable,
+    arguments,
+    workingDirectory: workingDirectory,
+    environment: environment,
+    evalOutput: true,
+  );
+  return manager.evalStdout();
+}
+
 /// Starts a process using the [executable], passing it [arguments].
 ///
 /// Returns a process manager that decorates the process with extra

lib/web_ui/lib/src/engine/canvaskit/initialization.dart

@@ -59,6 +59,11 @@ const bool canvasKitForceCpuOnly = bool.fromEnvironment(
     'FLUTTER_WEB_CANVASKIT_FORCE_CPU_ONLY',
     defaultValue: false);
 
+/// The version of CanvasKit used by the web engine by default.
+// DO NOT EDIT THE NEXT LINE OF CODE MANUALLY
+// See `lib/web_ui/README.md` for how to roll CanvasKit to a new version.
+const String canvaskitVersion = '0.28.1';
+
 /// The URL to use when downloading the CanvasKit script and associated wasm.
 ///
 /// The expected directory structure nested under this URL is as follows:
@@ -86,7 +91,7 @@ const bool canvasKitForceCpuOnly = bool.fromEnvironment(
 /// NPM, update this URL to `https://unpkg.com/canvaskit-wasm@0.34.0/bin/`.
 const String canvasKitBaseUrl = String.fromEnvironment(
   'FLUTTER_WEB_CANVASKIT_URL',
-  defaultValue: 'https://unpkg.com/canvaskit-wasm@0.28.1/bin/',
+  defaultValue: 'https://unpkg.com/canvaskit-wasm@$canvaskitVersion/bin/',
 );
 const String canvasKitBuildUrl =
     canvasKitBaseUrl + (kProfileMode ? 'profiling/' : '');

tools/licenses/lib/main.dart

@@ -1878,10 +1878,23 @@ class _RepositoryRootThirdPartyDirectory extends _RepositoryGenericThirdPartyDir
       return _RepositoryVulkanDirectory(this, entry);
     if (entry.name == 'wuffs')
       return _RepositoryWuffsDirectory(this, entry);
+    if (entry.name == 'web_dependencies')
+      return _RepositoryThirdPartyWebDependenciesDirectory(this, entry);
     return super.createSubdirectory(entry);
   }
 }
 
+/// Corresponds to the `src/third_party/web_dependencies` directory.
+class _RepositoryThirdPartyWebDependenciesDirectory extends _RepositoryDirectory {
+  _RepositoryThirdPartyWebDependenciesDirectory(_RepositoryDirectory parent, fs.Directory io) : super(parent, io);
+
+  @override
+  bool shouldRecurse(fs.IoNode entry) {
+    return entry.name != 'canvaskit' // redundant; covered by Skia dependencies
+        && super.shouldRecurse(entry);
+  }
+}
+
 class _RepositoryBoringSSLThirdPartyDirectory extends _RepositoryDirectory {
   _RepositoryBoringSSLThirdPartyDirectory(_RepositoryDirectory parent, fs.Directory io) : super(parent, io);