Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Pipeline] Multithreaded build #4450

Closed
wants to merge 8 commits into from
+457 −98
Closed

[Pipeline] Multithreaded build #4450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
790748f
Pass Logger instance per BuildEvent
nkast Sep 18, 2016
e2d0d2d
Thread-safe GetProcessorDefaultValues()
nkast Sep 18, 2016
1684b5c
Thread-safe pipelineBuildEvents
nkast Sep 18, 2016
4f10a4a
Register processing PipelineBuildEvent
nkast Sep 18, 2016
45fddcf
Add ConsoleAsyncLogger
nkast Sep 18, 2016
d393a8c
Add BuildAsyncState
nkast Sep 18, 2016
15cccad
BuildContent in parallel
nkast Sep 18, 2016
d1bdc4e
[Pipeline] Add /MultithreadMode switch
nkast Sep 18, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions Build/Projects/MGCB.definition
View file
Open in desktop
Expand Up @@ -35,7 +35,9 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="CommandLineParser.cs" />
<Compile Include="BuildContent.cs" />
<Compile Include="BuildAsyncState.cs" />
<Compile Include="ConsoleLogger.cs" />
<Compile Include="ConsoleAsyncLogger.cs" />
<Compile Include="SourceFileCollection.cs" />
<None Include="app.config" />

Expand Down
3 changes: 3 additions & 0 deletions MonoGame.Framework.Content.Pipeline/Builder/PipelineBuildEvent.cs
View file
Open in desktop
Expand Up @@ -68,6 +68,9 @@ public PipelineBuildEvent()
[XmlIgnore]
public OpaqueDataDictionary Parameters { get; set; }

[XmlIgnore]
public ContentBuildLogger Logger { get; set; }

public class Pair
{
public string Key { get; set; }
Expand Down
7 changes: 5 additions & 2 deletions MonoGame.Framework.Content.Pipeline/Builder/PipelineImporterContext.cs
View file
Open in desktop
Expand Up @@ -10,14 +10,17 @@ public class PipelineImporterContext : ContentImporterContext
{
private readonly PipelineManager _manager;

public PipelineImporterContext(PipelineManager manager)
private readonly PipelineBuildEvent _pipelineEvent;

public PipelineImporterContext(PipelineManager manager, PipelineBuildEvent pipelineEvent)
{
_manager = manager;
_pipelineEvent = pipelineEvent;
}

public override string IntermediateDirectory { get { return _manager.IntermediateDirectory; } }
public override string OutputDirectory { get { return _manager.OutputDirectory; } }
public override ContentBuildLogger Logger { get { return _manager.Logger; } }
public override ContentBuildLogger Logger { get { return _pipelineEvent.Logger; } }

public override void AddDependency(string filename)
{
Expand Down
141 changes: 88 additions & 53 deletions MonoGame.Framework.Content.Pipeline/Builder/PipelineManager.cs
View file
Open in desktop
Expand Up @@ -54,6 +54,8 @@ private struct ProcessorInfo
// Value = processor parameters
private readonly Dictionary<string, OpaqueDataDictionary> _processorDefaultValues;

private readonly SortedSet<string> _processingBuildEvents;

public string ProjectDirectory { get; private set; }
public string OutputDirectory { get; private set; }
public string IntermediateDirectory { get; private set; }
Expand Down Expand Up @@ -94,6 +96,7 @@ public PipelineManager(string projectDir, string outputDir, string intermediateD
{
_pipelineBuildEvents = new Dictionary<string, List<PipelineBuildEvent>>();
_processorDefaultValues = new Dictionary<string, OpaqueDataDictionary>();
_processingBuildEvents = new SortedSet<string>();
RethrowExceptions = true;

Assemblies = new List<string>();
Expand Down Expand Up @@ -398,27 +401,30 @@ public OpaqueDataDictionary GetProcessorDefaultValues(string processorName)
processorName = string.Empty;

OpaqueDataDictionary defaultValues;
if (!_processorDefaultValues.TryGetValue(processorName, out defaultValues))
lock (_processorDefaultValues)
{
// Create the content processor instance and read the default values.
defaultValues = new OpaqueDataDictionary();
var processorType = GetProcessorType(processorName);
if (processorType != null)
if (!_processorDefaultValues.TryGetValue(processorName, out defaultValues))
{
try
{
var processor = (IContentProcessor)Activator.CreateInstance(processorType);
var properties = processorType.GetProperties(BindingFlags.FlattenHierarchy | BindingFlags.Public | BindingFlags.Instance);
foreach (var property in properties)
defaultValues.Add(property.Name, property.GetValue(processor, null));
}
catch
// Create the content processor instance and read the default values.
defaultValues = new OpaqueDataDictionary();
var processorType = GetProcessorType(processorName);
if (processorType != null)
{
// Ignore exception. Will be handled in ProcessContent.
try
{
var processor = (IContentProcessor)Activator.CreateInstance(processorType);
var properties = processorType.GetProperties(BindingFlags.FlattenHierarchy | BindingFlags.Public | BindingFlags.Instance);
foreach (var property in properties)
defaultValues.Add(property.Name, property.GetValue(processor, null));
}
catch
{
// Ignore exception. Will be handled in ProcessContent.
}
}
}

_processorDefaultValues.Add(processorName, defaultValues);
_processorDefaultValues.Add(processorName, defaultValues);
}
}

return defaultValues;
Expand Down Expand Up @@ -525,7 +531,7 @@ public void RegisterContent(string sourceFilepath, string outputFilepath = null,
TrackPipelineBuildEvent(contentEvent);
}

public PipelineBuildEvent BuildContent(string sourceFilepath, string outputFilepath = null, string importerName = null, string processorName = null, OpaqueDataDictionary processorParameters = null)
public PipelineBuildEvent BuildContent(ContentBuildLogger logger, string sourceFilepath, string outputFilepath = null, string importerName = null, string processorName = null, OpaqueDataDictionary processorParameters = null)
{
sourceFilepath = PathHelper.Normalize(sourceFilepath);
ResolveOutputFilepath(sourceFilepath, ref outputFilepath);
Expand All @@ -540,6 +546,7 @@ public PipelineBuildEvent BuildContent(string sourceFilepath, string outputFilep
Importer = importerName,
Processor = processorName,
Parameters = ValidateProcessorParameters(processorName, processorParameters),
Logger = logger
};

// Load the previous content event if it exists.
Expand All @@ -559,21 +566,23 @@ private void BuildContent(PipelineBuildEvent pipelineEvent, PipelineBuildEvent c
throw new PipelineException("The source file '{0}' does not exist!", pipelineEvent.SourceFile);
}

Logger.PushFile(pipelineEvent.SourceFile);
pipelineEvent.Logger.PushFile(pipelineEvent.SourceFile);

// Keep track of all build events. (Required to resolve automatic names "AssetName_n".)
TrackPipelineBuildEvent(pipelineEvent);

var rebuild = pipelineEvent.NeedsRebuild(this, cachedEvent);
var building = RegisterBuildEvent(pipelineEvent);
var rebuild = pipelineEvent.NeedsRebuild(this, cachedEvent);
rebuild = rebuild && !building;
if (rebuild)
Logger.LogMessage("{0}", pipelineEvent.SourceFile);
pipelineEvent.Logger.LogMessage("{0}", pipelineEvent.SourceFile);
else
Logger.LogMessage("Skipping {0}", pipelineEvent.SourceFile);
pipelineEvent.Logger.LogMessage("Skipping {0}", pipelineEvent.SourceFile);

Logger.Indent();
pipelineEvent.Logger.Indent();
try
{
if (!rebuild)
if (!rebuild && cachedEvent != null)
{
// While this asset doesn't need to be rebuilt the dependent assets might.
foreach (var asset in cachedEvent.BuildAsset)
Expand All @@ -596,6 +605,7 @@ private void BuildContent(PipelineBuildEvent pipelineEvent, PipelineBuildEvent c
Importer = assetCachedEvent.Importer,
Processor = assetCachedEvent.Processor,
Parameters = assetCachedEvent.Parameters,
Logger = pipelineEvent.Logger,
};

// Give the asset a chance to rebuild.
Expand All @@ -622,11 +632,24 @@ private void BuildContent(PipelineBuildEvent pipelineEvent, PipelineBuildEvent c
}
finally
{
Logger.Unindent();
Logger.PopFile();
pipelineEvent.Logger.Unindent();
pipelineEvent.Logger.PopFile();
}
}

private bool RegisterBuildEvent(PipelineBuildEvent pipelineEvent)
{
lock (_processingBuildEvents)
{
if (!_processingBuildEvents.Contains(pipelineEvent.DestFile))
{
_processingBuildEvents.Add(pipelineEvent.DestFile);
return false;
}
}
return true;
}

public object ProcessContent(PipelineBuildEvent pipelineEvent)
{
if (!File.Exists(pipelineEvent.SourceFile))
Expand All @@ -647,7 +670,7 @@ public object ProcessContent(PipelineBuildEvent pipelineEvent)
{
try
{
var importContext = new PipelineImporterContext(this);
var importContext = new PipelineImporterContext(this, pipelineEvent);
importedObject = importer.Import(pipelineEvent.SourceFile, importContext);
}
catch (PipelineException)
Expand All @@ -661,7 +684,7 @@ public object ProcessContent(PipelineBuildEvent pipelineEvent)
}
else
{
var importContext = new PipelineImporterContext(this);
var importContext = new PipelineImporterContext(this, pipelineEvent);
importedObject = importer.Import(pipelineEvent.SourceFile, importContext);
}

Expand Down Expand Up @@ -762,7 +785,10 @@ public void CleanContent(string sourceFilepath, string outputFilepath = null)
// Remove event file (.mgcontent file) from intermediate folder.
FileHelper.DeleteIfExists(eventFilepath);

_pipelineBuildEvents.Remove(sourceFilepath);
lock (_pipelineBuildEvents)
{
_pipelineBuildEvents.Remove(sourceFilepath);
}
}

private void WriteXnb(object content, PipelineBuildEvent pipelineEvent)
Expand Down Expand Up @@ -791,15 +817,18 @@ private void WriteXnb(object content, PipelineBuildEvent pipelineEvent)
private void TrackPipelineBuildEvent(PipelineBuildEvent pipelineEvent)
{
List<PipelineBuildEvent> pipelineBuildEvents;
bool eventsFound = _pipelineBuildEvents.TryGetValue(pipelineEvent.SourceFile, out pipelineBuildEvents);
if (!eventsFound)
lock (_pipelineBuildEvents)
{
pipelineBuildEvents = new List<PipelineBuildEvent>();
_pipelineBuildEvents.Add(pipelineEvent.SourceFile, pipelineBuildEvents);
}
bool eventsFound = _pipelineBuildEvents.TryGetValue(pipelineEvent.SourceFile, out pipelineBuildEvents);
if (!eventsFound)
{
pipelineBuildEvents = new List<PipelineBuildEvent>();
_pipelineBuildEvents.Add(pipelineEvent.SourceFile, pipelineBuildEvents);
}

if (FindMatchingEvent(pipelineBuildEvents, pipelineEvent.DestFile, pipelineEvent.Importer, pipelineEvent.Processor, pipelineEvent.Parameters) == null)
pipelineBuildEvents.Add(pipelineEvent);
if (FindMatchingEvent(pipelineBuildEvents, pipelineEvent.DestFile, pipelineEvent.Importer, pipelineEvent.Processor, pipelineEvent.Parameters) == null)
pipelineBuildEvents.Add(pipelineEvent);
}
}

/// <summary>
Expand All @@ -810,7 +839,7 @@ private void TrackPipelineBuildEvent(PipelineBuildEvent pipelineEvent)
/// <param name="processorName">The name of the content processor. Can be <see langword="null"/>.</param>
/// <param name="processorParameters">The processor parameters. Can be <see langword="null"/>.</param>
/// <returns>The asset name.</returns>
public string GetAssetName(string sourceFileName, string importerName, string processorName, OpaqueDataDictionary processorParameters)
public string GetAssetName(ContentBuildLogger logger, string sourceFileName, string importerName, string processorName, OpaqueDataDictionary processorParameters)
{
Debug.Assert(Path.IsPathRooted(sourceFileName), "Absolute path expected.");

Expand All @@ -819,23 +848,26 @@ public string GetAssetName(string sourceFileName, string importerName, string pr
string relativeSourceFileName = PathHelper.GetRelativePath(ProjectDirectory, sourceFileName);

List<PipelineBuildEvent> pipelineBuildEvents;
if (_pipelineBuildEvents.TryGetValue(sourceFileName, out pipelineBuildEvents))
lock (_pipelineBuildEvents)
{
// This source file has already been build.
// --> Compare pipeline build events.
ResolveImporterAndProcessor(sourceFileName, ref importerName, ref processorName);

var matchingEvent = FindMatchingEvent(pipelineBuildEvents, null, importerName, processorName, processorParameters);
if (matchingEvent != null)
if (_pipelineBuildEvents.TryGetValue(sourceFileName, out pipelineBuildEvents))
{
// Matching pipeline build event found.
string existingName = matchingEvent.DestFile;
existingName = PathHelper.GetRelativePath(OutputDirectory, existingName);
existingName = existingName.Substring(0, existingName.Length - 4); // Remove ".xnb".
return existingName;
}
// This source file has already been build.
// --> Compare pipeline build events.
ResolveImporterAndProcessor(sourceFileName, ref importerName, ref processorName);

var matchingEvent = FindMatchingEvent(pipelineBuildEvents, null, importerName, processorName, processorParameters);
if (matchingEvent != null)
{
// Matching pipeline build event found.
string existingName = matchingEvent.DestFile;
existingName = PathHelper.GetRelativePath(OutputDirectory, existingName);
existingName = existingName.Substring(0, existingName.Length - 4); // Remove ".xnb".
return existingName;
}

Logger.LogMessage(string.Format("Warning: Asset {0} built multiple times with different settings.", relativeSourceFileName));
logger.LogMessage(string.Format("Warning: Asset {0} built multiple times with different settings.", relativeSourceFileName));
}
}

// No pipeline build event with matching settings found.
Expand Down Expand Up @@ -910,9 +942,12 @@ private bool IsAssetNameUsed(string assetName)
{
string destFile = Path.Combine(OutputDirectory, assetName + ".xnb");

return _pipelineBuildEvents.SelectMany(pair => pair.Value)
.Select(pipelineEvent => pipelineEvent.DestFile)
.Any(existingDestFile => destFile.Equals(existingDestFile, StringComparison.OrdinalIgnoreCase));
lock (_pipelineBuildEvents)
{
return _pipelineBuildEvents.SelectMany(pair => pair.Value)
.Select(pipelineEvent => pipelineEvent.DestFile)
.Any(existingDestFile => destFile.Equals(existingDestFile, StringComparison.OrdinalIgnoreCase));
}
}
}
}
8 changes: 4 additions & 4 deletions MonoGame.Framework.Content.Pipeline/Builder/PipelineProcessorContext.cs
View file
Open in desktop
Expand Up @@ -32,7 +32,7 @@ public PipelineProcessorContext(PipelineManager manager, PipelineBuildEvent pipe

public override OpaqueDataDictionary Parameters { get { return _pipelineEvent.Parameters; } }

public override ContentBuildLogger Logger { get { return _manager.Logger; } }
public override ContentBuildLogger Logger { get { return _pipelineEvent.Logger; } }

public override void AddDependency(string filename)
{
Expand All @@ -49,7 +49,7 @@ public override void AddOutputFile(string filename)
OpaqueDataDictionary processorParameters)
{
var processor = _manager.CreateProcessor(processorName, processorParameters);
var processContext = new PipelineProcessorContext(_manager, new PipelineBuildEvent { Parameters = processorParameters } );
var processContext = new PipelineProcessorContext(_manager, new PipelineBuildEvent { Parameters = processorParameters, Logger = this.Logger } );
var processedObject = processor.Process(input, processContext);

// Add its dependencies and built assets to ours.
Expand Down Expand Up @@ -96,10 +96,10 @@ public override void AddOutputFile(string filename)
string assetName)
{
if (string.IsNullOrEmpty(assetName))
assetName = _manager.GetAssetName(sourceAsset.Filename, importerName, processorName, processorParameters);
assetName = _manager.GetAssetName(Logger, sourceAsset.Filename, importerName, processorName, processorParameters);

// Build the content.
var buildEvent = _manager.BuildContent(sourceAsset.Filename, assetName, importerName, processorName, processorParameters);
var buildEvent = _manager.BuildContent(Logger, sourceAsset.Filename, assetName, importerName, processorName, processorParameters);

// Record that we built this dependent asset.
_pipelineEvent.BuildAsset.AddUnique(buildEvent.DestFile);
Expand Down
10 changes: 5 additions & 5 deletions MonoGame.Framework.Content.Pipeline/ContentBuildLogger.cs
View file
Open in desktop
Expand Up @@ -22,7 +22,7 @@ public abstract class ContentBuildLogger
/// <summary>
/// Gets or sets the base reference path used when reporting errors during the content build process.
/// </summary>
public string LoggerRootDirectory { get; set; }
public virtual string LoggerRootDirectory { get; set; }

/// <summary>
/// Initializes a new instance of ContentBuildLogger.
Expand Down Expand Up @@ -99,7 +99,7 @@ ContentIdentity contentIdentity
/// <summary>
/// Outputs a message indicating that a content asset has completed processing.
/// </summary>
public void PopFile()
public virtual void PopFile()
{
filenames.Pop();
}
Expand All @@ -109,17 +109,17 @@ public void PopFile()
/// All logger warnings or error exceptions from this time forward to the next PopFile call refer to this file.
/// </summary>
/// <param name="filename">Name of the file containing future messages.</param>
public void PushFile(string filename)
public virtual void PushFile(string filename)
{
filenames.Push(filename);
}

public void Indent()
public virtual void Indent()
{
indentCount++;
}

public void Unindent()
public virtual void Unindent()
{
indentCount--;
}
Expand Down
13 changes: 13 additions & 0 deletions Tools/MGCB/BuildAsyncState.cs
View file
Open in desktop
@@ -0,0 +1,13 @@
using Microsoft.Xna.Framework.Content.Pipeline;

namespace MGCB
{
internal class BuildAsyncState
{
public string SourceFile { get; internal set; }
public string Importer { get; internal set; }
public string Processor { get; internal set; }
public OpaqueDataDictionary ProcessorParams { get; internal set; }
public ConsoleAsyncLogger Logger { get; internal set; }
}
}