Skip to content

Commit

Permalink
[tool] Generate a binary version of the asset manifest (#117233)
Browse files Browse the repository at this point in the history 
* initial

* update asset_bundle_package_test

* Update asset_bundle_test.dart

* Update asset_bundle_package_fonts_test.dart

* update pubspec checksum for smc dependency

* flutter update-packages --force-upgrade

* prefer += 1 over ++

Co-authored-by: Jonah Williams <jonahwilliams@google.com>

* add regexp comment

* rescope int list comparison function

* update packages

Co-authored-by: Jonah Williams <jonahwilliams@google.com>
  • Loading branch information
@andrewkolos @jonahwilliams
andrewkolos and jonahwilliams committed Jan 12, 2023
1 parent 40bc6b5 commit 5630d53
Show file tree
Hide file tree
Showing 8 changed files with 231 additions and 96 deletions.
4 changes: 2 additions & 2 deletions dev/benchmarks/multiple_flutters/module/pubspec.yaml
View file
Expand Up @@ -41,7 +41,7 @@ dependencies:
typed_data: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
vector_math: 2.1.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
win32: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
xdg_directories: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
xdg_directories: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

flutter:
uses-material-design: true
Expand All @@ -51,4 +51,4 @@ flutter:
androidPackage: com.example.multiple_flutters_module
iosBundleIdentifier: com.example.multipleFluttersModule

# PUBSPEC CHECKSUM: e9fc
# PUBSPEC CHECKSUM: eafd
4 changes: 2 additions & 2 deletions dev/integration_tests/android_views/pubspec.yaml
View file
Expand Up @@ -47,7 +47,7 @@ dependencies:
vm_service: 9.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
webdriver: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
win32: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
xdg_directories: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
xdg_directories: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

dev_dependencies:
flutter_test:
Expand Down Expand Up @@ -94,4 +94,4 @@ dev_dependencies:
flutter:
uses-material-design: true

# PUBSPEC CHECKSUM: 11b0
# PUBSPEC CHECKSUM: 28b1
4 changes: 2 additions & 2 deletions dev/integration_tests/hybrid_android_views/pubspec.yaml
View file
Expand Up @@ -45,7 +45,7 @@ dependencies:
vm_service: 9.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
webdriver: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
win32: 3.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
xdg_directories: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
xdg_directories: 0.2.0+3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

dev_dependencies:
flutter_test:
Expand Down Expand Up @@ -92,4 +92,4 @@ dev_dependencies:
flutter:
uses-material-design: true

# PUBSPEC CHECKSUM: 11b0
# PUBSPEC CHECKSUM: 28b1
122 changes: 99 additions & 23 deletions packages/flutter_tools/lib/src/asset.dart
View file
Expand Up @@ -2,8 +2,11 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:typed_data';

import 'package:meta/meta.dart';
import 'package:package_config/package_config.dart';
import 'package:standard_message_codec/standard_message_codec.dart';

import 'base/context.dart';
import 'base/deferred_component.dart';
Expand Down Expand Up @@ -162,7 +165,11 @@ class ManifestAssetBundle implements AssetBundle {

DateTime? _lastBuildTimestamp;

static const String _kAssetManifestJson = 'AssetManifest.json';
// We assume the main asset is designed for a device pixel ratio of 1.0.
static const double _defaultResolution = 1.0;
static const String _kAssetManifestJsonFilename = 'AssetManifest.json';
static const String _kAssetManifestBinFilename = 'AssetManifest.bin';

static const String _kNoticeFile = 'NOTICES';
// Comically, this can't be name with the more common .gz file extension
// because when it's part of an AAR and brought into another APK via gradle,
Expand Down Expand Up @@ -230,8 +237,15 @@ class ManifestAssetBundle implements AssetBundle {
// device.
_lastBuildTimestamp = DateTime.now();
if (flutterManifest.isEmpty) {
entries[_kAssetManifestJson] = DevFSStringContent('{}');
entryKinds[_kAssetManifestJson] = AssetKind.regular;
entries[_kAssetManifestJsonFilename] = DevFSStringContent('{}');
entryKinds[_kAssetManifestJsonFilename] = AssetKind.regular;
entries[_kAssetManifestJsonFilename] = DevFSStringContent('{}');
entryKinds[_kAssetManifestJsonFilename] = AssetKind.regular;
final ByteData emptyAssetManifest =
const StandardMessageCodec().encodeMessage(<dynamic, dynamic>{})!;
entries[_kAssetManifestBinFilename] =
DevFSByteContent(emptyAssetManifest.buffer.asUint8List(0, emptyAssetManifest.lengthInBytes));
entryKinds[_kAssetManifestBinFilename] = AssetKind.regular;
return 0;
}

Expand Down Expand Up @@ -428,7 +442,10 @@ class ManifestAssetBundle implements AssetBundle {
_wildcardDirectories[uri] ??= _fileSystem.directory(uri);
}

final DevFSStringContent assetManifest = _createAssetManifest(assetVariants, deferredComponentsAssetVariants);
final Map<String, List<String>> assetManifest =
_createAssetManifest(assetVariants, deferredComponentsAssetVariants);
final DevFSStringContent assetManifestJson = DevFSStringContent(json.encode(assetManifest));
final DevFSByteContent assetManifestBinary = _createAssetManifestBinary(assetManifest);
final DevFSStringContent fontManifest = DevFSStringContent(json.encode(fonts));
final LicenseResult licenseResult = _licenseCollector.obtainLicenses(packageConfig, additionalLicenseFiles);
if (licenseResult.errorMessages.isNotEmpty) {
Expand All @@ -452,26 +469,40 @@ class ManifestAssetBundle implements AssetBundle {
_fileSystem.file('DOES_NOT_EXIST_RERUN_FOR_WILDCARD$suffix').absolute);
}

_setIfChanged(_kAssetManifestJson, assetManifest, AssetKind.regular);
_setIfChanged(_kAssetManifestJsonFilename, assetManifestJson, AssetKind.regular);
_setIfChanged(_kAssetManifestBinFilename, assetManifestBinary, AssetKind.regular);
_setIfChanged(kFontManifestJson, fontManifest, AssetKind.regular);
_setLicenseIfChanged(licenseResult.combinedLicenses, targetPlatform);
return 0;
}

@override
List<File> additionalDependencies = <File>[];

void _setIfChanged(String key, DevFSStringContent content, AssetKind assetKind) {
if (!entries.containsKey(key)) {
entries[key] = content;
entryKinds[key] = assetKind;
void _setIfChanged(String key, DevFSContent content, AssetKind assetKind) {
final DevFSContent? oldContent = entries[key];
// In the case that the content is unchanged, we want to avoid an overwrite
// as the isModified property may be reset to true,
if (oldContent is DevFSByteContent && content is DevFSByteContent &&
_compareIntLists(oldContent.bytes, content.bytes)) {
return;
}
final DevFSStringContent? oldContent = entries[key] as DevFSStringContent?;
if (oldContent?.string != content.string) {
entries[key] = content;
entryKinds[key] = assetKind;

entries[key] = content;
entryKinds[key] = assetKind;
}

static bool _compareIntLists(List<int> o1, List<int> o2) {
if (o1.length != o2.length) {
return false;
}

for (int index = 0; index < o1.length; index++) {
if (o1[index] != o2[index]) {
return false;
}
}

return true;
}

void _setLicenseIfChanged(
Expand Down Expand Up @@ -623,39 +654,84 @@ class ManifestAssetBundle implements AssetBundle {
return deferredComponentsAssetVariants;
}

DevFSStringContent _createAssetManifest(
Map<String, List<String>> _createAssetManifest(
Map<_Asset, List<_Asset>> assetVariants,
Map<String, Map<_Asset, List<_Asset>>> deferredComponentsAssetVariants
) {
final Map<String, List<String>> jsonObject = <String, List<String>>{};
final Map<_Asset, List<String>> jsonEntries = <_Asset, List<String>>{};
final Map<String, List<String>> manifest = <String, List<String>>{};
final Map<_Asset, List<String>> entries = <_Asset, List<String>>{};
assetVariants.forEach((_Asset main, List<_Asset> variants) {
jsonEntries[main] = <String>[
entries[main] = <String>[
for (final _Asset variant in variants)
variant.entryUri.path,
];
});
if (deferredComponentsAssetVariants != null) {
for (final Map<_Asset, List<_Asset>> componentAssets in deferredComponentsAssetVariants.values) {
componentAssets.forEach((_Asset main, List<_Asset> variants) {
jsonEntries[main] = <String>[
entries[main] = <String>[
for (final _Asset variant in variants)
variant.entryUri.path,
];
});
}
}
final List<_Asset> sortedKeys = jsonEntries.keys.toList()
final List<_Asset> sortedKeys = entries.keys.toList()
..sort((_Asset left, _Asset right) => left.entryUri.path.compareTo(right.entryUri.path));
for (final _Asset main in sortedKeys) {
final String decodedEntryPath = Uri.decodeFull(main.entryUri.path);
final List<String> rawEntryVariantsPaths = jsonEntries[main]!;
final List<String> rawEntryVariantsPaths = entries[main]!;
final List<String> decodedEntryVariantPaths = rawEntryVariantsPaths
.map((String value) => Uri.decodeFull(value))
.toList();
jsonObject[decodedEntryPath] = decodedEntryVariantPaths;
manifest[decodedEntryPath] = decodedEntryVariantPaths;
}
return manifest;
}

// Matches path-like strings ending in a number followed by an 'x'.
// Example matches include "assets/animals/2.0x", "plants/3x", and "2.7x".
static final RegExp _extractPixelRatioFromKeyRegExp = RegExp(r'/?(\d+(\.\d*)?)x$');

DevFSByteContent _createAssetManifestBinary(
Map<String, List<String>> assetManifest
) {
double parseScale(String key) {
final Uri assetUri = Uri.parse(key);
String directoryPath = '';
if (assetUri.pathSegments.length > 1) {
directoryPath = assetUri.pathSegments[assetUri.pathSegments.length - 2];
}

final Match? match = _extractPixelRatioFromKeyRegExp.firstMatch(directoryPath);
if (match != null && match.groupCount > 0) {
return double.parse(match.group(1)!);
}
return _defaultResolution;
}
return DevFSStringContent(json.encode(jsonObject));

final Map<String, dynamic> result = <String, dynamic>{};

for (final MapEntry<String, dynamic> manifestEntry in assetManifest.entries) {
final List<dynamic> resultVariants = <dynamic>[];
final List<String> entries = (manifestEntry.value as List<dynamic>).cast<String>();
for (final String variant in entries) {
if (variant == manifestEntry.key) {
// With the newer binary format, don't include the main asset in it's
// list of variants. This reduces parsing time at runtime.
continue;
}
final Map<String, dynamic> resultVariant = <String, dynamic>{};
final double variantDevicePixelRatio = parseScale(variant);
resultVariant['asset'] = variant;
resultVariant['dpr'] = variantDevicePixelRatio;
resultVariants.add(resultVariant);
}
result[manifestEntry.key] = resultVariants;
}

final ByteData message = const StandardMessageCodec().encodeMessage(result)!;
return DevFSByteContent(message.buffer.asUint8List(0, message.lengthInBytes));
}

/// Prefixes family names and asset paths of fonts included from packages with
Expand Down
4 changes: 3 additions & 1 deletion packages/flutter_tools/pubspec.yaml
View file
Expand Up @@ -57,6 +57,8 @@ dependencies:

vm_service: 9.4.0

standard_message_codec: 0.0.1+3

_fe_analyzer_shared: 52.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
analyzer: 5.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
boolean_selector: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
Expand Down Expand Up @@ -102,4 +104,4 @@ dartdoc:
# Exclude this package from the hosted API docs.
nodoc: true

# PUBSPEC CHECKSUM: 02f9
# PUBSPEC CHECKSUM: 899b
4 changes: 2 additions & 2 deletions packages/flutter_tools/test/general.shard/asset_bundle_package_fonts_test.dart
View file
Expand Up @@ -111,8 +111,8 @@ $fontsSection

final AssetBundle bundle = AssetBundleFactory.instance.createBundle();
await bundle.build(packagesPath: '.packages');
expect(bundle.entries.length, 3); // LICENSE, AssetManifest, FontManifest
expect(bundle.entries.containsKey('FontManifest.json'), isTrue);
expect(bundle.entries.keys, unorderedEquals(<String>['AssetManifest.bin',
'AssetManifest.json', 'FontManifest.json', 'NOTICES.Z']));
}, overrides: <Type, Generator>{
FileSystem: () => testFileSystem,
ProcessManager: () => FakeProcessManager.any(),
Expand Down

0 comments on commit 5630d53

Please sign in to comment.