Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Address F# Assembly Resolution Issues #893

Closed
wants to merge 16 commits into from
Closed

Address F# Assembly Resolution Issues #893

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
159e37c
add twilio test
dsyme Nov 8, 2016
0d20f6b
function renaming
dsyme Nov 8, 2016
5fbc7ff
add nuget references test
dsyme Nov 8, 2016
a0b687e
Address F# Assembly Resolution Issues
sylvanc Nov 8, 2016
2a7090f
delete temporary dll and pdb after compilation
sylvanc Nov 9, 2016
f5e81d2
Merge branch 'dsyme-888' into fsharp-assembly-resolve
sylvanc Nov 9, 2016
31e7b3c
Merge branch 'dsyme-891' into fsharp-assembly-resolve
sylvanc Nov 9, 2016
d22c965
Update run.csx
dsyme Nov 11, 2016
f5aeaf4
Update function.json
dsyme Nov 11, 2016
6e6a926
Update run.csx
dsyme Nov 11, 2016
ad90d51
add new tests to csproj
sylvanc Nov 11, 2016
b1f21cb
put twilio reference back in run.csx
sylvanc Nov 11, 2016
785a86c
fix file references
sylvanc Nov 11, 2016
40053e2
add function.json for new tests
sylvanc Nov 11, 2016
05093fa
add project.json to NugetChartingReferences
sylvanc Nov 11, 2016
4395106
disabled NugetChartingReferences
sylvanc Nov 11, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
55 changes: 43 additions & 12 deletions src/WebJobs.Script/Description/DotNet/FSharp/FSharpCompilationService.cs
View file
Open in desktop
Expand Up @@ -6,6 +6,7 @@
using System.IO;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Scripting;
Expand All @@ -25,11 +26,13 @@ internal class FSharpCompiler : ICompilationService
= new Lazy<InteractiveAssemblyLoader>(() => new InteractiveAssemblyLoader(), LazyThreadSafetyMode.ExecutionAndPublication);

private readonly OptimizationLevel _optimizationLevel;
private readonly Regex _hashRRegex;

public FSharpCompiler(IFunctionMetadataResolver metadataResolver, OptimizationLevel optimizationLevel)
{
_metadataResolver = metadataResolver;
_optimizationLevel = optimizationLevel;
_hashRRegex = new Regex(@"^\s*#r\s+", RegexOptions.Compiled | RegexOptions.IgnoreCase);
}

public string Language
Expand All @@ -50,8 +53,28 @@ public IEnumerable<string> SupportedFileTypes

public ICompilation GetFunctionCompilation(FunctionMetadata functionMetadata)
{
// First use the C# compiler to resolve references, to get consistenct with the C# Azure Functions programming model
Script<object> script = CodeAnalysis.CSharp.Scripting.CSharpScript.Create("using System;", options: _metadataResolver.CreateScriptOptions(), assemblyLoader: AssemblyLoader.Value);
// First use the C# compiler to resolve references, to get consistency with the C# Azure Functions programming model
// Add the #r statements from the .fsx file to the resolver source
string scriptSource = GetFunctionSource(functionMetadata);
var resolverSourceBuilder = new StringBuilder();

using (StringReader sr = new StringReader(scriptSource))
{
string line;

while ((line = sr.ReadLine()) != null)
{
if (_hashRRegex.IsMatch(line))
{
resolverSourceBuilder.AppendLine(line);
}
}
}

resolverSourceBuilder.AppendLine("using System;");
var resolverSource = resolverSourceBuilder.ToString();

Script<object> script = CodeAnalysis.CSharp.Scripting.CSharpScript.Create(resolverSource, options: _metadataResolver.CreateScriptOptions(), assemblyLoader: AssemblyLoader.Value);
Compilation compilation = script.GetCompilation();

var compiler = new SimpleSourceCodeServices(msbuildEnabled: FSharpOption<bool>.Some(false));
Expand All @@ -60,6 +83,10 @@ public ICompilation GetFunctionCompilation(FunctionMetadata functionMetadata)
FSharpOption<Assembly> assemblyOption = null;
string scriptFilePath = Path.Combine(Path.GetTempPath(), Path.GetFileName(functionMetadata.ScriptFile));

var asmName = FunctionAssemblyLoader.GetAssemblyNameFromMetadata(functionMetadata, compilation.AssemblyName);
var dllName = Path.GetTempPath() + asmName + ".dll";
var pdbName = Path.ChangeExtension(dllName, "pdb");

try
{
var scriptFileBuilder = new StringBuilder();
Expand All @@ -77,7 +104,6 @@ public ICompilation GetFunctionCompilation(FunctionMetadata functionMetadata)
scriptFileBuilder.AppendLine("# 0 @\"" + functionMetadata.ScriptFile + "\"");

// Add our original script
string scriptSource = GetFunctionSource(functionMetadata);
scriptFileBuilder.AppendLine(scriptSource);

File.WriteAllText(scriptFilePath, scriptFileBuilder.ToString());
Expand Down Expand Up @@ -131,8 +157,7 @@ public ICompilation GetFunctionCompilation(FunctionMetadata functionMetadata)
otherFlags.Add("--lib:" + Path.Combine(Path.GetDirectoryName(functionMetadata.ScriptFile), DotNetConstants.PrivateAssembliesFolderName));
}

// This output DLL isn't actually written by FSharp.Compiler.Service when CompileToDynamicAssembly is called
otherFlags.Add("--out:" + Path.ChangeExtension(Path.GetTempFileName(), "dll"));
otherFlags.Add("--out:" + dllName);

// Get the #load closure
FSharpChecker checker = FSharpChecker.Create(null, null, null, msbuildEnabled: FSharpOption<bool>.Some(false));
Expand All @@ -149,18 +174,24 @@ public ICompilation GetFunctionCompilation(FunctionMetadata functionMetadata)
// Add the (adjusted) script file itself
otherFlags.Add(scriptFilePath);

// Make the output streams (unused)
var outStreams = FSharpOption<Tuple<TextWriter, TextWriter>>.Some(new Tuple<TextWriter, TextWriter>(Console.Out, Console.Error));

// Compile the script to a dynamic assembly
var result = compiler.CompileToDynamicAssembly(otherFlags: otherFlags.ToArray(), execute: outStreams);

// Compile the script to a static assembly
var result = compiler.Compile(otherFlags.ToArray());
errors = result.Item1;
assemblyOption = result.Item3;
var code = result.Item2;

if (code == 0)
{
var assemblyBytes = File.ReadAllBytes(dllName);
var pdbBytes = File.ReadAllBytes(pdbName);
var assembly = Assembly.Load(assemblyBytes, pdbBytes);
assemblyOption = FSharpOption<Assembly>.Some(assembly);
}
}
finally
{
File.Delete(scriptFilePath);
File.Delete(dllName);
File.Delete(pdbName);
}
return new FSharpCompilation(errors, assemblyOption);
}
Expand Down
42 changes: 27 additions & 15 deletions src/WebJobs.Script/Description/DotNet/FunctionAssemblyLoader.cs
View file
Open in desktop
Expand Up @@ -19,7 +19,8 @@ public class FunctionAssemblyLoader : IDisposable
{
// Prefix that uniquely identifies our assemblies
// i.e.: "ƒ-<functionname>"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: update (or remove) the comment

public const string AssemblyPrefix = "\u0192-";
public const string AssemblyPrefix = "f-";
public const string AssemblySeparator = "__";

private readonly ConcurrentDictionary<string, FunctionAssemblyLoadContext> _functionContexts = new ConcurrentDictionary<string, FunctionAssemblyLoadContext>();
private readonly Regex _functionNameFromAssemblyRegex;
Expand All @@ -29,7 +30,7 @@ public FunctionAssemblyLoader(string rootScriptPath)
{
_rootScriptUri = new Uri(rootScriptPath, UriKind.RelativeOrAbsolute);
AppDomain.CurrentDomain.AssemblyResolve += ResolveAssembly;
_functionNameFromAssemblyRegex = new Regex(string.Format(CultureInfo.InvariantCulture, "^{0}(?<name>.*?)#", AssemblyPrefix), RegexOptions.Compiled);
_functionNameFromAssemblyRegex = new Regex(string.Format(CultureInfo.InvariantCulture, "^{0}(?<name>.*?){1}", AssemblyPrefix, AssemblySeparator), RegexOptions.Compiled);
}

public void Dispose()
Expand All @@ -51,22 +52,33 @@ internal Assembly ResolveAssembly(object sender, ResolveEventArgs args)
FunctionAssemblyLoadContext context = GetFunctionContext(args.RequestingAssembly);
Assembly result = null;

if (context != null)
try
{
result = context.ResolveAssembly(args.Name);
}
if (context != null)
{
result = context.ResolveAssembly(args.Name);
}

// If we were unable to resolve the assembly, apply the current App Domain policy and attempt to load it.
// This allows us to correctly handle retargetable assemblies, redirects, etc.
if (result == null)
// If we were unable to resolve the assembly, apply the current App Domain policy and attempt to load it.
// This allows us to correctly handle retargetable assemblies, redirects, etc.
if (result == null)
{
string assemblyName = ((AppDomain)sender).ApplyPolicy(args.Name);

// If after applying the current policy, we now have a different target assembly name, attempt to load that
// assembly
if (string.Compare(assemblyName, args.Name) != 0)
{
result = Assembly.Load(assemblyName);
}
}
}
catch (Exception e)
{
string assemblyName = ((AppDomain)sender).ApplyPolicy(args.Name);

// If after applying the current policy, we now have a different target assembly name, attempt to load that
// assembly
if (string.Compare(assemblyName, args.Name) != 0)
if (context != null)
{
result = Assembly.Load(assemblyName);
context.TraceWriter.Warning(string.Format(CultureInfo.InvariantCulture,
"Exception during runtime resolution of assembly '{0}': '{1}'", args.Name, e.ToString()));
}
}

Expand Down Expand Up @@ -165,7 +177,7 @@ private FunctionAssemblyLoadContext GetFunctionContext(string functionName)

public static string GetAssemblyNameFromMetadata(FunctionMetadata metadata, string suffix)
{
return AssemblyPrefix + metadata.Name + "#" + suffix;
return AssemblyPrefix + metadata.Name + AssemblySeparator + suffix.GetHashCode().ToString();
}

public string GetFunctionNameFromAssembly(Assembly assembly)
Expand Down
6 changes: 6 additions & 0 deletions test/WebJobs.Script.Tests/CSharpEndToEndTests.cs
View file
Open in desktop
Expand Up @@ -29,6 +29,12 @@ public async Task ServiceBusQueueTriggerToBlobTest()
await ServiceBusQueueTriggerToBlobTestImpl();
}

[Fact]
public async Task TwilioReferenceInvokeSucceeds()
{
await TwilioReferenceInvokeSucceedsImpl(isDotNet: true);
}

[Fact]
public async Task MobileTables()
{
Expand Down
20 changes: 20 additions & 0 deletions test/WebJobs.Script.Tests/EndToEndTestsBase.cs
View file
Open in desktop
Expand Up @@ -157,6 +157,26 @@ public async Task QueueTriggerToBlobTest()
Assert.True(trace.Replace(" ", string.Empty).Contains(messageContent.Replace(" ", string.Empty)));
}

protected async Task TwilioReferenceInvokeSucceedsImpl(bool isDotNet)
{
if (isDotNet)
{
TestHelpers.ClearFunctionLogs("TwilioReference");

string testData = Guid.NewGuid().ToString();
string inputName = "input";
Dictionary<string, object> arguments = new Dictionary<string, object>
{
{ inputName, testData }
};
await Fixture.Host.CallAsync("TwilioReference", arguments);

// make sure the input string made it all the way through
var logs = await TestHelpers.GetFunctionLogsAsync("TwilioReference");
Assert.True(logs.Any(p => p.Contains(testData)));
}
}

protected async Task DocumentDBTest()
{
// DocumentDB tests need the following environment vars:
Expand Down
24 changes: 24 additions & 0 deletions test/WebJobs.Script.Tests/FSharpEndToEndTests.cs
View file
Open in desktop
Expand Up @@ -27,6 +27,12 @@ public async Task ServiceBusQueueTriggerToBlobTest()
await ServiceBusQueueTriggerToBlobTestImpl();
}

[Fact]
public async Task TwilioReferenceInvokeSucceeds()
{
await TwilioReferenceInvokeSucceedsImpl(isDotNet: true);
}

//[Fact]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove commented out code

//public async Task MobileTables()
//{
Expand Down Expand Up @@ -172,6 +178,24 @@ public async Task SharedAssemblyDependenciesAreLoaded()
Assert.Equal("secondary type value", request.Properties["DependencyOutput"]);
}

//[Fact]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove commented out code

//public async Task NugetChartingReferencesInvokeSucceeds()
//{
// TestHelpers.ClearFunctionLogs("NugetChartingReferences");

// string testData = Guid.NewGuid().ToString();
// string inputName = "input";
// Dictionary<string, object> arguments = new Dictionary<string, object>
// {
// { inputName, testData }
// };
// await Fixture.Host.CallAsync("NugetChartingReferences", arguments);

// // make sure the input string made it all the way through
// var logs = await TestHelpers.GetFunctionLogsAsync("NugetChartingReferences");
// Assert.True(logs.Any(p => p.Contains(testData)));
//}

[Fact]
public async Task PrivateAssemblyDependenciesAreLoaded()
{
Expand Down
9 changes: 9 additions & 0 deletions test/WebJobs.Script.Tests/TestScripts/CSharp/TwilioReference/function.json
View file
Open in desktop
@@ -0,0 +1,9 @@
{
"bindings": [
{
"type": "manualTrigger",
"name": "input",
"direction": "in"
}
]
}
10 changes: 10 additions & 0 deletions test/WebJobs.Script.Tests/TestScripts/CSharp/TwilioReference/run.csx
View file
Open in desktop
@@ -0,0 +1,10 @@
#r "Twilio.Api"

using System;
using Microsoft.Azure.WebJobs.Host;
using Twilio;

public static void Run(string input, TraceWriter log)
{
log.Info(input);
}
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/ApiHubFileSender/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you should just remove all of these !COMPILED blocks from all the sample/test functions. You don't have to block on this if you feel strongly, but I think it adds more confusion/bloat than it is worth. Furthermore, even with your update I don't get any intellisense - I think your paths are still wrong :) My vote is to just remove and keep the functions clean.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If kept, I believe we're going going too far up and the path should be ../../src/WebJobs.Script.Host/bin/Debug

#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/ApiHubFileTrigger/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/ApiHubTable/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/ApiHubTableClient/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/ApiHubTableEntity/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/AssembliesFromSharedLocation/run.fsx
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

#if !COMPILED
open System.Runtime.InteropServices
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/DocumentDBIn/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/DocumentDBOut/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/LoadScriptReference/run.fsx
View file
Open in desktop
Expand Up @@ -3,7 +3,7 @@

#if !COMPILED
open System.Runtime.InteropServices
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/ManualTrigger/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/MobileTableIn/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down
2 changes: 1 addition & 1 deletion test/WebJobs.Script.Tests/TestScripts/FSharp/MobileTableOut/run.fsx
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@
// This prelude allows scripts to be edited in Visual Studio or another F# editing environment

#if !COMPILED
#I "../../../../../bin/Binaries/WebJobs.Script.Host"
#I "../../../../../src/WebJobs.Script.Host/bin/Debug"
#r "Microsoft.Azure.WebJobs.Host.dll"
#r "Microsoft.Azure.WebJobs.Extensions.dll"
#endif
Expand Down