This repository has been archived by the owner on Jan 23, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 219
Implement Single-file Bundler #5286
Merged
Merged
Changes from all commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
cf5a946
Implement Single-file Bundler
swaroop-sridhar 37339a8
Name bundler as Microsoft.DotNet.Build.Bundle
swaroop-sridhar 79c3171
Address more code review feedback
swaroop-sridhar d1c9f82
Address feedback by @vitek-karas
swaroop-sridhar 94eab1d
Fix a test build dependency
swaroop-sridhar 6d5a1b2
Address feedback by @0xd4d
swaroop-sridhar ef96034
Fix file sorting while embedding to disk.
swaroop-sridhar 8e2e112
Update Command line switches
swaroop-sridhar bd26f90
Handle Subdirectories
swaroop-sridhar 762eea7
Test: Change bundle/extraction directory
swaroop-sridhar File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 20 additions & 0 deletions
20
src/managed/Microsoft.DotNet.Build.Bundle/BundleException.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Copyright (c) .NET Foundation and contributors. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
|
|
||
| namespace Microsoft.DotNet.Build.Bundle | ||
| { | ||
| /// <summary> | ||
| /// This exception is thrown when a bundle/extraction | ||
| /// operation fails due known user errors. | ||
| /// </summary> | ||
| public class BundleException : Exception | ||
| { | ||
| public BundleException(string message) : | ||
| base(message) | ||
| { | ||
| } | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,220 @@ | ||
| // Copyright (c) .NET Foundation and contributors. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
| using System.IO; | ||
| using System.Reflection.PortableExecutable; | ||
|
|
||
| namespace Microsoft.DotNet.Build.Bundle | ||
| { | ||
| /// <summary> | ||
| /// Bundler: Functionality to embed the managed app and its dependencies | ||
| /// into the host native binary. | ||
| /// </summary> | ||
| public class Bundler | ||
| { | ||
| string HostName; | ||
| string SourceDir; | ||
| string OutputDir; | ||
| bool EmbedPDBs; | ||
|
|
||
| string Application; | ||
| string DepsJson; | ||
| string RuntimeConfigJson; | ||
| string RuntimeConfigDevJson; | ||
|
|
||
| /// <summary> | ||
| /// Align embedded assemblies such that they can be loaded | ||
| /// directly from memory-mapped bundle. | ||
| /// TBD: Set the correct value of alignment while working on | ||
| /// the runtime changes to load the embedded assemblies. | ||
| /// </summary> | ||
| const int AssemblyAlignment = 16; | ||
|
|
||
| public static string Version => (Manifest.MajorVersion + "." + Manifest.MinorVersion); | ||
|
|
||
| public Bundler(string hostName, string sourceDir, string outputDir, bool embedPDBs) | ||
| { | ||
| SourceDir = sourceDir; | ||
| OutputDir = outputDir; | ||
| HostName = hostName; | ||
| EmbedPDBs = embedPDBs; | ||
| } | ||
|
|
||
| void ValidateFiles() | ||
| { | ||
| // Check required directories | ||
| if (!Directory.Exists(SourceDir)) | ||
| { | ||
| throw new BundleException("Dirctory not found: " + SourceDir); | ||
| } | ||
| if (!Directory.Exists(OutputDir)) | ||
| { | ||
| throw new BundleException("Dirctory not found: " + OutputDir); | ||
| } | ||
|
|
||
| // Set default names | ||
| string baseName = Path.GetFileNameWithoutExtension(HostName); | ||
| Application = baseName + ".dll"; | ||
| DepsJson = baseName + ".deps.json"; | ||
| RuntimeConfigJson = baseName + ".runtimeconfig.json"; | ||
| RuntimeConfigDevJson = baseName + ".runtimeconfig.dev.json"; | ||
|
|
||
| // Check that required files exist on disk. | ||
| Action<string> checkFileExists = (string name) => | ||
| { | ||
| string path = Path.Combine(SourceDir, name); | ||
| if (!File.Exists(path)) | ||
| { | ||
| throw new BundleException("File not found: " + path); | ||
| } | ||
| }; | ||
|
|
||
| checkFileExists(HostName); | ||
| checkFileExists(Application); | ||
| // The *.json files may or may not exist. | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Embed 'file' into 'bundle' | ||
| /// </summary> | ||
| /// <returns>Returns the offset of the start 'file' within 'bundle'</returns> | ||
|
|
||
| long AddToBundle(Stream bundle, Stream file, FileType type = FileType.Extract) | ||
| { | ||
| // Allign assemblies, since they are loaded directly from bundle | ||
| if (type == FileType.Assembly) | ||
| { | ||
| long misalignment = (bundle.Position % AssemblyAlignment); | ||
|
|
||
| if (misalignment != 0) | ||
| { | ||
| long padding = AssemblyAlignment - misalignment; | ||
| bundle.Position += padding; | ||
| } | ||
| } | ||
|
|
||
| file.Position = 0; | ||
| long startOffset = bundle.Position; | ||
| file.CopyTo(bundle); | ||
|
|
||
| return startOffset; | ||
| } | ||
|
|
||
| bool ShouldEmbed(string fileRelativePath) | ||
| { | ||
| if (fileRelativePath.Equals(HostName)) | ||
| { | ||
| // The bundle starts with the host, so ignore it while embedding. | ||
| return false; | ||
| } | ||
|
|
||
| if (fileRelativePath.Equals(RuntimeConfigDevJson)) | ||
| { | ||
| // Ignore the machine specific configuration file. | ||
| return false; | ||
| } | ||
|
|
||
| if (Path.GetExtension(fileRelativePath).ToLower().Equals(".pdb")) | ||
| { | ||
| return EmbedPDBs; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| FileType InferType(string fileRelativePath, Stream file) | ||
| { | ||
| if (fileRelativePath.Equals(DepsJson)) | ||
| { | ||
| return FileType.DepsJson; | ||
| } | ||
|
|
||
| if (fileRelativePath.Equals(RuntimeConfigJson)) | ||
| { | ||
| return FileType.RuntimeConfigJson; | ||
| } | ||
|
|
||
| if (fileRelativePath.Equals(Application)) | ||
| { | ||
| return FileType.Application; | ||
| } | ||
|
|
||
| try | ||
| { | ||
| PEReader peReader = new PEReader(file); | ||
| CorHeader corHeader = peReader.PEHeaders.CorHeader; | ||
| if ((corHeader != null) && ((corHeader.Flags & CorFlags.ILOnly) != 0)) | ||
| { | ||
| return FileType.Assembly; | ||
| } | ||
| } | ||
| catch (BadImageFormatException) | ||
| { | ||
| } | ||
|
|
||
| return FileType.Extract; | ||
| } | ||
|
|
||
| void GenerateBundle() | ||
| { | ||
| string bundlePath = Path.Combine(OutputDir, HostName); | ||
|
|
||
| if (File.Exists(bundlePath)) | ||
| { | ||
| Program.Log($"Ovewriting existing File {bundlePath}"); | ||
| } | ||
|
|
||
| // Start with a copy of the host executable. | ||
| // Copy the file to preserve its permissions. | ||
| File.Copy(Path.Combine(SourceDir, HostName), bundlePath, overwrite: true); | ||
|
|
||
| using (BinaryWriter writer = new BinaryWriter(File.OpenWrite(bundlePath))) | ||
| { | ||
| Stream bundle = writer.BaseStream; | ||
| Manifest manifest = new Manifest(); | ||
|
|
||
| bundle.Position = bundle.Length; | ||
| int sourceDirLen = Path.GetFullPath(SourceDir).Length + 1; | ||
|
|
||
| // Get all files in the source directory and all sub-directories. | ||
| string[] sources = Directory.GetFiles(SourceDir, searchPattern: "*", searchOption: SearchOption.AllDirectories); | ||
|
|
||
| // Sort the file names to keep the bundle construction deterministic. | ||
| Array.Sort(sources, StringComparer.Ordinal); | ||
|
|
||
| foreach (string filePath in sources) | ||
| { | ||
| // filePath is the full-path of files within source directory, and any of its sub-directories. | ||
| // We only need the relative paths with respect to the source directory. | ||
| string relativePath = filePath.Substring(sourceDirLen); | ||
|
|
||
| if (!ShouldEmbed(relativePath)) | ||
| { | ||
| Program.Log($"Skip: {relativePath}"); | ||
| continue; | ||
| } | ||
|
|
||
| using (FileStream file = File.OpenRead(filePath)) | ||
| { | ||
| FileType type = InferType(relativePath, file); | ||
| long startOffset = AddToBundle(bundle, file, type); | ||
| FileEntry entry = new FileEntry(type, relativePath, startOffset, file.Length); | ||
| manifest.Files.Add(entry); | ||
| Program.Log($"Embed: {entry}"); | ||
| } | ||
| } | ||
|
|
||
| manifest.Write(writer); | ||
| Program.Log($"Bundle: Path={bundlePath} Size={bundle.Length}"); | ||
| } | ||
| } | ||
|
|
||
| public void MakeBundle() | ||
| { | ||
| ValidateFiles(); | ||
| GenerateBundle(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| // Copyright (c) .NET Foundation and contributors. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
| using System.IO; | ||
|
|
||
| namespace Microsoft.DotNet.Build.Bundle | ||
| { | ||
| /// <summary> | ||
| /// Extractor: The functionality to extract the files embedded | ||
| /// within a bundle to sepearte files. | ||
| /// </summary> | ||
| public class Extractor | ||
| { | ||
| string OutputDir; | ||
| string BundlePath; | ||
|
|
||
| public Extractor(string bundlePath, string outputDir) | ||
| { | ||
| BundlePath = bundlePath; | ||
| OutputDir = outputDir; | ||
| } | ||
|
|
||
| public void Spill() | ||
| { | ||
| try | ||
| { | ||
| if (!File.Exists(BundlePath)) | ||
| { | ||
| throw new BundleException("File not found: " + BundlePath); | ||
| } | ||
|
|
||
| using (BinaryReader reader = new BinaryReader(File.OpenRead(BundlePath))) | ||
| { | ||
| Manifest manifest = Manifest.Read(reader); | ||
|
|
||
| foreach (FileEntry entry in manifest.Files) | ||
| { | ||
| Program.Log($"Spill: {entry}"); | ||
|
|
||
| string fileRelativePath = entry.RelativePath.Replace(Manifest.DirectorySeparatorChar, Path.DirectorySeparatorChar); | ||
| string filePath = Path.Combine(OutputDir, fileRelativePath); | ||
| string fileDir = Path.GetDirectoryName(filePath); | ||
|
|
||
| if ((fileDir != null) && !fileDir.Equals(String.Empty)) | ||
| { | ||
| Directory.CreateDirectory(fileDir); | ||
| } | ||
|
|
||
| reader.BaseStream.Position = entry.Offset; | ||
| using (BinaryWriter file = new BinaryWriter(File.Create(filePath))) | ||
| { | ||
| long size = entry.Size; | ||
| do | ||
| { | ||
| int copySize = (int)(size % int.MaxValue); | ||
| file.Write(reader.ReadBytes(copySize)); | ||
| size -= copySize; | ||
| } while (size > 0); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| catch (IOException) | ||
| { | ||
| throw new BundleException("Malformed Bundle"); | ||
| } | ||
| catch (ArgumentOutOfRangeException) | ||
| { | ||
| // Trying to set file-stream position to an invalid value | ||
| throw new BundleException("Malformed Bundle"); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: indentation of comments