Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
d6dbef6
Change GetModuleFilePaths() to include a modules directory in root of…
davidobrien1985 Nov 10, 2016
96a3616
Create AddToModulePaths() method to avoid code duplication.
davidobrien1985 Nov 10, 2016
a478b40
Change AddToModulePaths() to accept array of strings to further reduc…
davidobrien1985 Nov 11, 2016
e69dd06
Change AddToModulePaths so that it searches the modules folder recurs…
davidobrien1985 Nov 12, 2016
0781847
Change GetModuleFilePaths() to include a modules directory in root of…
davidobrien1985 Nov 10, 2016
f1aacf9
Create AddToModulePaths() method to avoid code duplication.
davidobrien1985 Nov 10, 2016
7f148f8
Change AddToModulePaths() to accept array of strings to further reduc…
davidobrien1985 Nov 11, 2016
f680ed6
Change AddToModulePaths so that it searches the modules folder recurs…
davidobrien1985 Nov 12, 2016
32d4726
Merge branch 'feature/powershell_module_root' of https://github.com/d…
davidobrien1985 Nov 12, 2016
25f7fa0
Change GetModuleFilePaths() to include a modules directory in root of…
davidobrien1985 Nov 10, 2016
e54f4ad
Create AddToModulePaths() method to avoid code duplication.
davidobrien1985 Nov 10, 2016
98629d0
Change AddToModulePaths() to accept array of strings to further reduc…
davidobrien1985 Nov 11, 2016
9c742aa
Change AddToModulePaths so that it searches the modules folder recurs…
davidobrien1985 Nov 12, 2016
ea7f409
Change GetModuleFilePaths() to include a modules directory in root of…
davidobrien1985 Nov 10, 2016
da35501
Create AddToModulePaths() method to avoid code duplication.
davidobrien1985 Nov 10, 2016
3fb4187
Change AddToModulePaths() to accept array of strings to further reduc…
davidobrien1985 Nov 11, 2016
62390bc
Merge branch 'feature/powershell_module_root' of https://github.com/d…
davidobrien1985 Nov 15, 2016
eb824c0
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Nov 15, 2016
bc0b1af
Merge branch 'dev' of https://github.com/Azure/azure-webjobs-sdk-scri…
davidobrien1985 Nov 15, 2016
fae756b
Change GetModuleFilePaths() from for loop to foreach.
davidobrien1985 Nov 18, 2016
733b7ca
Merge branch 'dev' of https://github.com/Azure/azure-webjobs-sdk-scri…
davidobrien1985 Nov 18, 2016
4babc58
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Nov 26, 2016
7bc6206
Add test for modules dir in root
davidobrien1985 Nov 26, 2016
8d579e0
Set reference to RootTestModules in current scope
davidobrien1985 Nov 26, 2016
e996fb8
Change AddToModulePaths method input to accept params to make calling…
davidobrien1985 Nov 27, 2016
6b6904c
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Nov 29, 2016
1eaecd9
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Nov 30, 2016
55f06a1
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Nov 30, 2016
d55e774
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 5, 2016
8d170b1
Add code to identify duplicate modules and ignore duplicates by name
davidobrien1985 Dec 6, 2016
75e915e
Add tests for FindDuplicateModules()
davidobrien1985 Dec 6, 2016
b61a4be
Merge dev into feature/powershell_module_root
davidobrien1985 Dec 7, 2016
7598e6f
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 7, 2016
5e757d2
CreateModuleFiles() now supports modules Dir in root of App so that G…
davidobrien1985 Dec 7, 2016
350a413
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 10, 2016
96b7f62
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 13, 2016
936f00b
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 13, 2016
8d97263
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 15, 2016
1f6179c
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 20, 2016
c6b1af5
Merge branch 'dev' into feature/powershell_module_root
davidobrien1985 Dec 22, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ private async Task<PSDataCollection<ErrorRecord>> InvokePowerShellScript(Diction
System.Management.Automation.PowerShell.Create())
{
powerShellInstance.Runspace = runspace;
_moduleFiles = GetModuleFilePaths(_host.ScriptConfig.RootScriptPath, _functionName);
_moduleFiles = FindDuplicateModules(GetModuleFilePaths(_host.ScriptConfig.RootScriptPath, _functionName), _host.ScriptConfig.RootScriptPath, _functionName);
// Remove duplicates from _moduleFiles and only keep the ones closest to the Function in moduleDirectory

if (_moduleFiles.Any())
{
powerShellInstance.AddCommand("Import-Module").AddArgument(_moduleFiles);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So we run an auto import on every single module we discover. For a modules folder scoped to a single function (the way we work currently) that is OK, since you only add modules you want to use in your function anyways.

However, in this PR you're adding a shared modules folder that sits above all functions, and all the modules in there will be added to all functions. That seems problematic. Can't you do what you're trying to do today by adding explicit import statements to pull in the shared modules you need from a shared folder? That avoids the code duplication issue, and also allows you to pull only a subset of shared modules in as needed.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mathewc this would be #897
Right now I just want to have a "super modules" directory that does not sit within one function.
Let's say I have one module that I need to share across 5 functions in one Function App I only want it in one place, not in 5.
The next step will be to add implicit import as described in above issue which will address the issue you are having right now.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But can't you simply put your shared module in a shared folder, and add implicit import statement to each function already? As you pointed out in #897 auto import has performance implications. While the problem is not too bad for function level modules (since you have to explicitly add them to your function dir so intent is pretty clear), for cross function modules auto importing into every function doesn't feel right. Isn't simply adding the proper import statements to each of your functions a cleaner approach? Are there any issues blocking you from doing that today?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, I'd prefer to not have any auto-import, at all, not even on a function level.
I was told that auto-import should stay so to not break current behaviour.
I would either opt for removing auto-import everywhere or add my approach here and auto-import.
How does it work in other languages on Functions?
Having implicit imports in one case and explicit in other cases makes it quite confusing, unless we start working with PSModulePath.
PowerShell users will either expect auto loading to just work, wherever they put stuff or not.

One option would be to add the two "modules" locations to PSModulePath and use PowerShell's auto-loading of modules, but this again would have performance implications.
In the end, PowerShell on Functions will never perform as well as the other languages.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should hold off on doing any auto-import from a shared folder across functions. People can get the desired behavior in a better more predictable way today by using explicit references.

That said, I do think the small fix you mad to make the module resolution handle sub directories is good. Your call on whether you want to scope to just that and get this in, or set that as a different PR. Thanks for your help and patience :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mathewc fair enough. If you like to cherry-pick the commits for the subdirectory code then I'm fine with that.
I'm also going to create a PR to remove auto-import altogether as I don't think that this is a great idea in general. This will address #897 then.

Expand Down Expand Up @@ -261,18 +263,58 @@ internal static List<string> GetModuleFilePaths(string rootScriptPath, string fu
{
List<string> modulePaths = new List<string>();
string functionFolder = Path.Combine(rootScriptPath, functionName);
string rootModuleDirectory = Path.Combine(rootScriptPath, PowerShellConstants.ModulesFolderName);
string moduleDirectory = Path.Combine(functionFolder, PowerShellConstants.ModulesFolderName);
if (Directory.Exists(moduleDirectory))
modulePaths.AddRange(AddToModulePaths(rootModuleDirectory, moduleDirectory));
return modulePaths;
}

internal static List<string> AddToModulePaths(params string[] directories)
{
List<string> paths = new List<string>();
foreach (string directory in directories)
{
modulePaths.AddRange(Directory.GetFiles(moduleDirectory,
PowerShellConstants.ModulesManifestFileExtensionPattern));
modulePaths.AddRange(Directory.GetFiles(moduleDirectory,
PowerShellConstants.ModulesBinaryFileExtensionPattern));
modulePaths.AddRange(Directory.GetFiles(moduleDirectory,
PowerShellConstants.ModulesScriptFileExtensionPattern));
if (Directory.Exists(directory))
{
paths.AddRange(Directory.GetFiles(directory,
PowerShellConstants.ModulesManifestFileExtensionPattern,
SearchOption.AllDirectories));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is the file and code for SearchOption?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Kindly also add tests for the following,

  1. nested folders
  2. ensure that resolution in case of duplicates is as expected.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tohling SearchOption is part of System.IO

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@fabiocav, ah, I missed Directory.GetFiles. Thanks!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tohling How would you want to handle "duplicates"? Usually I would say "closest to the function" wins.
However, PowerShell can import multiple versions of the same module into one environment. It is then up to the user to decide which one he/she wants to use. That said, because Azure Functions right now imports everything automatically, this won't necessarily work.
Hence #897

Copy link
Contributor

@tohling tohling Nov 21, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@davidobrien1985 Yes, I agree that the correct change is to allow the customer to decide which one to use even when there are duplicates. This is potentially a breaking change to existing customers who already have their scripts written with the current default behavior. There is a broader issue on how we can absorb your changes. We are working on addressing this. Kindly give us some time and I will get back to you.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tohling do you want me to continue working on this here then? Do you have an issue for this already here somewhere?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@davidobrien1985, yes you may continue to work on this. I don't have an issue opened yet. I really like the changes that you are making and my previous comment was just me thinking ahead on how soon they can be merged into the repo.

We truly value the work from contributors like yourself. We are discussing how to incorporate your changes in a timely and seamless manner for both you and customers who have already deployed their PowerShell Functions.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, added some logic to the tests and GetModuleFilePaths() is still passing.

In regards to duplicates:
I will assume, for now, that if there are duplicates (by filename) that the one closest in functionroot/modules is supposed to be imported.

paths.AddRange(Directory.GetFiles(directory,
PowerShellConstants.ModulesBinaryFileExtensionPattern,
SearchOption.AllDirectories));
paths.AddRange(Directory.GetFiles(directory,
PowerShellConstants.ModulesScriptFileExtensionPattern,
SearchOption.AllDirectories));
}
}
return paths;
}

return modulePaths;
internal static List<string> FindDuplicateModules(List<string> foundModules, string rootScriptPath, string functionName)
{
List<string> moduleFileNames = new List<string>();
string functionFolder = Path.Combine(rootScriptPath, functionName);
string rootModuleDirectory = Path.Combine(rootScriptPath, PowerShellConstants.ModulesFolderName);
string moduleDirectory = Path.Combine(functionFolder, PowerShellConstants.ModulesFolderName);

foreach (string entry in foundModules)
{
moduleFileNames.Add(Path.GetFileName(entry));
}

List<string> distinctModuleFiles = moduleFileNames.Distinct().ToList();

foreach (string distinctModuleFile in distinctModuleFiles)
{
string potentialModule = Path.Combine(moduleDirectory, distinctModuleFile);
string result = foundModules.SingleOrDefault(s => s == potentialModule);
if (result != null)
{
foundModules.Remove(Path.Combine(rootModuleDirectory, distinctModuleFile));
}
}
return foundModules;
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,17 @@ public void GetModuleFilePaths()
}
}

[Fact]
public void FindDuplicateModules()
{
List<string> actualModules = PowerShellFunctionInvoker.FindDuplicateModules(PowerShellFunctionInvoker.GetModuleFilePaths(_fixture.TestRootScriptPath, _fixture.TestFunctionName), _fixture.TestRootScriptPath, _fixture.TestFunctionName);
Assert.Equal(_fixture.ExpectedNumberOfImportedModules, actualModules.Count);
foreach (var actualModule in actualModules)
{
Assert.True(_fixture.TestModulesPath.Contains(actualModule));
}
}

[Fact]
public void GetRelativePath()
{
Expand Down Expand Up @@ -90,6 +101,9 @@ public Fixture()
TestRootScriptPath = Path.Combine(TestHelpers.FunctionsTestDirectory, "Functions");
TestFunctionRoot = Path.Combine(TestRootScriptPath, TestFunctionName);
TestModulesRoot = Path.Combine(TestFunctionRoot, "modules");
// The number of imported modules is expected to be 6, as the "manifest-module.psd1" in RootTestModules
// should be ignored when importing. So the 7 available modules should end up being 6.
ExpectedNumberOfImportedModules = 6;

TestScriptPath = CreateScriptFile(TestFunctionRoot);

Expand All @@ -100,7 +114,15 @@ public Fixture()
"manifest-module.psd1"
};

TestModulesPath = CreateModuleFiles(TestModulesRoot, TestModules);
RootTestModules = new string[]
{
"root-script-module.psm1",
"root-binary-module.dll",
"root-manifest-module.psd1",
"manifest-module.psd1"
};

TestModulesPath = CreateModuleFiles(TestModulesRoot, RootTestModules, TestModules);
}

public string TestFunctionRoot { get; private set; }
Expand All @@ -115,9 +137,12 @@ public Fixture()

public string[] TestModules { get; private set; }

public string[] RootTestModules { get; private set; }

public string TestModulesRoot { get; private set; }

public List<string> TestModulesPath { get; set; }
public int ExpectedNumberOfImportedModules { get; set; }

public void Dispose()
{
Expand Down Expand Up @@ -154,13 +179,20 @@ public string CreateScriptFile(string scriptRoot)
return path;
}

public List<string> CreateModuleFiles(string moduleRoot, string[] modules)
public List<string> CreateModuleFiles(string moduleRoot, string[] rootModules, string[] modules)
{
if (!Directory.Exists(moduleRoot))
{
Directory.CreateDirectory(moduleRoot);
}

string moduleDirinRoot = TestRootScriptPath + "\\modules";

if (!Directory.Exists(moduleDirinRoot))
{
Directory.CreateDirectory(moduleDirinRoot);
}

List<string> modulesPath = new List<string>();
foreach (var module in modules)
{
Expand All @@ -177,6 +209,21 @@ public List<string> CreateModuleFiles(string moduleRoot, string[] modules)
modulesPath.Add(path);
}

foreach (var rootModule in rootModules)
{
string path = Path.Combine(moduleDirinRoot, rootModule);
if (!File.Exists(path))
{
// Create a file to write to.
using (StreamWriter sw = File.CreateText(path))
{
sw.WriteLine(string.Format("This is a {0} file.", rootModule));
}
}

modulesPath.Add(path);
}

return modulesPath;
}
}
Expand Down