Skip to content
This repository has been archived by the owner on Mar 30, 2019. It is now read-only.

Commit

Permalink
[tkfxc] Add support for fast compilation when source file and include…
Browse files Browse the repository at this point in the history 
… files are not changed compare to the output file, using the /Ti command line switch to use this feature.
  • Loading branch information
@xoofx
xoofx committed Jan 5, 2013
1 parent c27dc6d commit ddbd2a6
Show file tree
Hide file tree
Showing 17 changed files with 2,665 additions and 2,509 deletions.
39 changes: 30 additions & 9 deletions Source/Toolkit/SharpDX.Toolkit.Compiler/EffectCompiler.cs
View file
Expand Up @@ -87,9 +87,9 @@ private EffectCompiler()
/// <param name="includeFileDelegate">The include file delegate.</param>
/// <returns>The result of compilation.</returns>
public static EffectCompilerResult CompileFromFile(string filePath, EffectCompilerFlags flags = EffectCompilerFlags.None, List<ShaderMacro> macros = null, List<string> includeDirectoryList = null,
IncludeFileDelegate includeFileDelegate = null)
IncludeFileDelegate includeFileDelegate = null, int destinationHashCode = 0)
{
return Compile(NativeFile.ReadAllText(filePath), filePath, flags, macros, includeDirectoryList, includeFileDelegate);
return Compile(NativeFile.ReadAllText(filePath), filePath, flags, macros, includeDirectoryList, includeFileDelegate, destinationHashCode);
}

/// <summary>
Expand All @@ -103,7 +103,7 @@ private EffectCompiler()
/// <param name="includeFileDelegate">The include file delegate.</param>
/// <returns>The result of compilation.</returns>
public static EffectCompilerResult Compile(string sourceCode, string filePath, EffectCompilerFlags flags = EffectCompilerFlags.None, List<ShaderMacro> macros = null, List<string> includeDirectoryList = null,
IncludeFileDelegate includeFileDelegate = null)
IncludeFileDelegate includeFileDelegate = null, int destinationHashCode = 0)
{
var compiler = new EffectCompiler
{
Expand All @@ -112,7 +112,7 @@ private EffectCompiler()
includeDirectoryList = includeDirectoryList ?? new List<string>(),
includeFileDelegate = includeFileDelegate
};
return new EffectCompilerResult(compiler.InternalCompile(sourceCode, filePath), compiler.logger);
return new EffectCompilerResult(compiler.InternalCompile(sourceCode, filePath, destinationHashCode), compiler.effectData, compiler.logger);
}

/// <summary>
Expand All @@ -125,25 +125,45 @@ public static string DisassembleShader(EffectData.Shader shader)
return new ShaderBytecode(shader.Bytecode).Disassemble(DisassemblyFlags.EnableColorCode | DisassemblyFlags.EnableInstructionNumbering);
}

private EffectData InternalCompile(string sourceCode, string fileName)
private bool InternalCompile(string sourceCode, string fileName, int destinationHashCode)
{
effectData = null;

fileName = fileName.Replace(@"\\", @"\");

logger = new EffectCompilerLogger();
var parser = new EffectParser { Logger = logger, IncludeFileCallback = includeFileDelegate};
parser.Macros.AddRange(macros);
parser.IncludeDirectoryList.AddRange(includeDirectoryList);

parserResult = parser.Parse(sourceCode, fileName);
if ((compilerFlags & EffectCompilerFlags.AllowFastCompileWithFileDependenciesChecking) != 0 && destinationHashCode != 0)
{
parserResult = parser.PrepareParsing(sourceCode, fileName);

if (parserResult.Hashcode == destinationHashCode)
{
return true;
}

parserResult = parser.ContinueParsing(parserResult);
}
else
{
parserResult = parser.Parse(sourceCode, fileName);
}

// Remove AllowFastCompileWithFileDependenciesChecking flags
compilerFlags &= ~EffectCompilerFlags.AllowFastCompileWithFileDependenciesChecking;

// Get back include handler.
includeHandler = parser.includeHandler;
includeHandler = parserResult.IncludeHandler;

if (logger.HasErrors)
return null;
return false;

effectData = new EffectData
{
HashCode = parserResult.Hashcode,
Effects = new List<EffectData.Effect>(),
Shaders = new List<EffectData.Shader>()
};
Expand All @@ -163,7 +183,8 @@ private EffectData InternalCompile(string sourceCode, string fileName)
HandleTechnique(techniqueAst);
}
}
return effectData;

return false;
}

private void HandleTechnique(Ast.Technique techniqueAst)
Expand Down
8 changes: 7 additions & 1 deletion Source/Toolkit/SharpDX.Toolkit.Compiler/EffectCompilerResult.cs
View file
Expand Up @@ -32,12 +32,18 @@ public sealed class EffectCompilerResult
/// </summary>
/// <param name="effectData">The EffectData.</param>
/// <param name="logger">The logger.</param>
internal EffectCompilerResult(EffectData effectData, Logger logger)
internal EffectCompilerResult(bool isUpToDate, EffectData effectData, Logger logger)
{
IsUpToDate = isUpToDate;
EffectData = effectData;
Logger = logger;
}

/// <summary>
/// A boolean indicating wether the compilation result is up to date (effectData == null and no need to perform a full compile)
/// </summary>
public readonly bool IsUpToDate;

/// <summary>
/// Gets the EffectData.
/// </summary>
Expand Down
187 changes: 71 additions & 116 deletions Source/Toolkit/SharpDX.Toolkit.Compiler/EffectParser.cs
View file
Expand Up @@ -44,33 +44,29 @@ internal class EffectParser
private Ast.Technique currentTechnique;
private Token currentToken;

internal IncludeHandler includeHandler;
private bool isPreviewToken = false;
private int parentCount = 0;
private List<Token> previewTokens = new List<Token>();
private EffectParserResult result;
private Token savedPreviewToken;
private IEnumerator<Token> tokenEnumerator;
private List<string> includeDirectoryList;


/// <summary>
/// Initializes a new instance of the <see cref="EffectParser" /> class.
/// </summary>
public EffectParser()
{
includeHandler = new IncludeHandler();
includeDirectoryList = new List<string>();
Macros = new List<ShaderMacro>();
}

/// <summary>
/// Gets or sets the include file callback.
/// </summary>
/// <value>The include file callback.</value>
public IncludeFileDelegate IncludeFileCallback
{
get { return includeHandler.IncludeFileCallback; }
set { includeHandler.IncludeFileCallback = value; }
}
public IncludeFileDelegate IncludeFileCallback { get; set; }

/// <summary>
/// Gets the macros.
Expand All @@ -84,7 +80,7 @@ public IncludeFileDelegate IncludeFileCallback
/// <value>The include directory list.</value>
public List<string> IncludeDirectoryList
{
get { return includeHandler.IncludeDirectories; }
get { return includeDirectoryList; }
}

/// <summary>
Expand All @@ -99,24 +95,45 @@ public List<string> IncludeDirectoryList
/// <param name="input">The input.</param>
/// <param name="fileName">Name of the file.</param>
/// <returns>Result of parsing</returns>
public EffectParserResult Parse(string input, string fileName)
public EffectParserResult PrepareParsing(string input, string fileName)
{
var filePath = Path.GetFullPath(fileName);
fileName = Path.GetFileName(fileName);

var localResult = new EffectParserResult
{
SourceFileName = Path.Combine(Environment.CurrentDirectory, filePath),
IncludeHandler = new FileIncludeHandler { Logger = Logger }
};
localResult.IncludeHandler.CurrentDirectory.Push(Path.GetDirectoryName(localResult.SourceFileName));
localResult.IncludeHandler.IncludeDirectories.AddRange(IncludeDirectoryList);
localResult.IncludeHandler.IncludeFileCallback = IncludeFileCallback;
localResult.IncludeHandler.FileResolved.Add(fileName, new FileIncludeHandler.FileItem(fileName, filePath, File.GetLastWriteTime(filePath)));

string compilationErrors = null;
var preprocessedInput = ShaderBytecode.Preprocess(input, Macros.ToArray(), localResult.IncludeHandler, out compilationErrors, fileName);
localResult.PreprocessedSource = preprocessedInput;

localResult.Hashcode = CalculateHashcode(localResult);

return localResult;
}

/// <summary>
/// Continues the parsing.
/// </summary>
/// <param name="previousParsing">The previous parsing.</param>
/// <returns>EffectParserResult.</returns>
public EffectParserResult ContinueParsing(EffectParserResult previousParsing)
{
// Reset count
parentCount = 0;
curlyBraceCount = 0;
bracketCount = 0;
result = new EffectParserResult {SourceFileName = Path.Combine(Environment.CurrentDirectory, Path.GetFullPath(fileName))};

includeHandler.Logger = Logger;
includeHandler.FileResolved.Clear();
includeHandler.CurrentDirectory.Clear();
includeHandler.CurrentDirectory.Push(Path.GetDirectoryName(result.SourceFileName));
result = previousParsing;

string compilationErrors = null;
var preprocessedInput = ShaderBytecode.Preprocess(input, Macros.ToArray(), includeHandler, out compilationErrors, fileName);
result.PreprocessedSource = preprocessedInput;

tokenEnumerator = Tokenizer.Run(preprocessedInput).GetEnumerator();
tokenEnumerator = Tokenizer.Run(previousParsing.PreprocessedSource).GetEnumerator();

do
{
Expand Down Expand Up @@ -151,10 +168,20 @@ public EffectParserResult Parse(string input, string fileName)
}
}


return result;
}

/// <summary>
/// Parses the specified input.
/// </summary>
/// <param name="input">The input.</param>
/// <param name="fileName">Name of the file.</param>
/// <returns>Result of parsing</returns>
public EffectParserResult Parse(string input, string fileName)
{
return ContinueParsing(PrepareParsing(input, fileName));
}

private bool CheckAllBracketsClosed()
{
return parentCount == 0 && bracketCount == 0 && curlyBraceCount == 0;
Expand Down Expand Up @@ -267,15 +294,10 @@ private Token NextToken()
currentFile = token.Value.Substring(1, token.Value.Length - 2);

// Replace "file" from #line preprocessor with the actual fullpath.
for (int i = 0; i < includeHandler.FileResolved.Count; i++)
var includeHandler = result.IncludeHandler;
if (includeHandler.FileResolved.ContainsKey(currentFile))
{
var fileResolved = includeHandler.FileResolved[i];
if (fileResolved.Item1 == currentFile)
{
currentFile = fileResolved.Item2;
includeHandler.FileResolved.RemoveAt(i);
break;
}
currentFile = includeHandler.FileResolved[currentFile].FilePath;
}
currentFile = currentFile.Replace(@"\\", @"\");

Expand Down Expand Up @@ -810,100 +832,33 @@ private Ast.ArrayInitializerExpression ParseArrayInitializerExpression(TokenType
return expression;
}

#region Nested type: IncludeHandler

internal class IncludeHandler : CallbackBase, Include
private int CalculateHashcode(EffectParserResult parserResult)
{
public readonly Stack<string> CurrentDirectory;

public readonly List<Tuple<string, string>> FileResolved;
public readonly List<string> IncludeDirectories;
public SourceSpan CurrentSpan;
public IncludeFileDelegate IncludeFileCallback;
public EffectCompilerLogger Logger;

public IncludeHandler()
{
IncludeDirectories = new List<string>();
CurrentDirectory = new Stack<string>();
FileResolved = new List<Tuple<string, string>>();
}

#region Include Members

public Stream Open(IncludeType type, string fileName, Stream parentStream)
var keys = new List<string>(parserResult.IncludeHandler.FileResolved.Keys);
keys.Sort(StringComparer.InvariantCultureIgnoreCase);

// Compute a HashCode using FNVModified
// based on filename and file write time
const uint p = 16777619;
uint hash = 2166136261;

foreach (var fileKey in keys)
{
var currentDirectory = CurrentDirectory.Peek();
if (currentDirectory == null)
currentDirectory = Environment.CurrentDirectory;

var filePath = fileName;

if (!Path.IsPathRooted(filePath))
{
var directoryToSearch = new List<string> {currentDirectory};
directoryToSearch.AddRange(IncludeDirectories);
foreach (var dirPath in directoryToSearch)
{
var selectedFile = Path.Combine(dirPath, fileName);
if (File.Exists(selectedFile))
{
filePath = selectedFile;
break;
}
}
}

Stream stream = null;

if (filePath == null || !File.Exists(filePath))
{
// Else try to use the include file callback
if (IncludeFileCallback != null)
{
stream = IncludeFileCallback(type == IncludeType.System, fileName);
if (stream != null)
{
FileResolved.Add(new Tuple<string, string>(fileName, fileName));
return stream;
}
}

Logger.Error("Unable to find file [{0}]", CurrentSpan, filePath ?? fileName);
return null;
}

stream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
CurrentDirectory.Push(Path.GetDirectoryName(Path.GetFullPath(filePath)));
FileResolved.Add(new Tuple<string, string>(fileName, Path.GetFullPath(filePath)));
return stream;
}
var fileItem = parserResult.IncludeHandler.FileResolved[fileKey];
var modifiedTime = fileItem.ModifiedTime;
var fileName = fileKey.ToLower();

public void Close(Stream stream)
{
stream.Close();
CurrentDirectory.Pop();
hash = (hash ^ (uint)fileName.GetHashCode()) * p;
hash = (hash ^ (uint)modifiedTime.GetHashCode()) * p;
}

#endregion
}

#endregion

#region Nested type: Tuple

internal class Tuple<T1, T2>
{
public T1 Item1;
public T2 Item2;

public Tuple(T1 item1, T2 item2)
{
Item1 = item1;
Item2 = item2;
}
hash += hash << 13;
hash ^= hash >> 7;
hash += hash << 3;
hash ^= hash >> 17;
hash += hash << 5;
return unchecked((int)hash);
}

#endregion
}
}
5 changes: 5 additions & 0 deletions Source/Toolkit/SharpDX.Toolkit.Compiler/EffectParserResult.cs
View file
Expand Up @@ -26,6 +26,11 @@ internal class EffectParserResult

public string PreprocessedSource;

public int Hashcode;

public Ast.Shader Shader;

public FileIncludeHandler IncludeHandler;

}
}

0 comments on commit ddbd2a6

Please sign in to comment.