Skip to content

Commit

Permalink
Implements a ForEach module that can be used to segment the processin…
Browse files Browse the repository at this point in the history 
…g of child modules per #47
  • Loading branch information
@daveaglick
daveaglick committed Aug 8, 2015
1 parent f86b6b0 commit 5ccc248
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 72 deletions.
2 changes: 1 addition & 1 deletion Wyam.Common/IExecutionContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,6 @@ public interface IExecutionContext

// This executes the specified modules with the specified input documents and returns the result documents
// If you pass in null for inputDocuments, a new input document with the initial metadata from the engine will be used
IReadOnlyList<IDocument> Execute(IEnumerable<IModule> modules, IEnumerable<IDocument> inputDocuments);
IReadOnlyList<IDocument> Execute(IEnumerable<IModule> modules, IEnumerable<IDocument> inputs);
}
}
24 changes: 6 additions & 18 deletions Wyam.Core/Modules/ContentModule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ public abstract class ContentModule : IModule
{
private readonly Func<IDocument, object> _content;
private readonly IModule[] _modules;
private bool _forEachDocument;

protected ContentModule(object content)
{
Expand All @@ -21,37 +20,26 @@ protected ContentModule(Func<IDocument, object> content)
_content = content ?? (x => null);
}

// For performance reasons, the specified modules will only be run once with a newly initialized, isolated document
// Otherwise, we'd need to run the whole set for each input document (I.e., multiple duplicate file reads, transformations, etc. for each input)
// If only one input document is available, it will be used as the initial document for the specified modules
// If more than one document is available, an empty initial document will be used
// To force usage of each input document in a set (I.e., A, B, and C input documents specify a unique "template" metadata value and you want to append
// some result of operating on that template value to each), make the content module a child of the ForEach module
// Each input will be applied against each result from the specified modules (I.e., if 2 inputs and the module chain results in 2 outputs, there will be 4 total outputs)
protected ContentModule(params IModule[] modules)
{
_modules = modules;
}

// Setting true for forEachDocument results in the whole sequence of modules being executed for every input document
// (as opposed to only being executed once with an empty initial document)
// You use this when the content modules rely on the input document - I.e., to read specific files based on metadata such as original file name in each input document
public IModule ForEachDocument()
{
_forEachDocument = true;
return this;
}

public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutionContext context)
{
if (_modules != null)
{
if (_forEachDocument)
{
return inputs.SelectMany(input => context.Execute(_modules, new[] { input }).SelectMany(result => Execute(result.Content, input, context)));
}
return context.Execute(_modules, null).SelectMany(result => inputs.SelectMany(input => Execute(result.Content, input, context)));
return context.Execute(_modules, inputs.Count == 1 ? inputs : null).SelectMany(x => inputs.SelectMany(y => Execute(x.Content, y, context)));
}
return inputs.SelectMany(x => Execute(_content(x), x, context));
}

// Note that content can be passed in as null, implementors should guard against that
// Note that content can be passed in as null, implementers should guard against that
protected abstract IEnumerable<IDocument> Execute(object content, IDocument input, IExecutionContext context);
}
}
6 changes: 3 additions & 3 deletions Wyam.Core/Modules/CopyFiles.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public CopyFiles(Func<IDocument, string> sourcePath)
{
if (sourcePath == null)
{
throw new ArgumentNullException("sourcePath");
throw new ArgumentNullException(nameof(sourcePath));
}

_sourcePath = sourcePath;
Expand All @@ -33,7 +33,7 @@ public CopyFiles(string searchPattern)
{
if (searchPattern == null)
{
throw new ArgumentNullException("searchPattern");
throw new ArgumentNullException(nameof(searchPattern));
}

_sourcePath = m => searchPattern;
Expand Down Expand Up @@ -68,7 +68,7 @@ public CopyFiles To(Func<string, string> destinationPath)
{
if (destinationPath == null)
{
throw new ArgumentNullException("destinationPath");
throw new ArgumentNullException(nameof(destinationPath));
}

_destinationPath = destinationPath;
Expand Down
26 changes: 26 additions & 0 deletions Wyam.Core/Modules/ForEach.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wyam.Common;

namespace Wyam.Core.Modules
{
// Executes the input documents one at a time against the specified modules
// The aggregated results of all child sequences is returned
public class ForEach : IModule
{
private readonly IModule[] _modules;

public ForEach(params IModule[] modules)
{
_modules = modules;
}

public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutionContext context)
{
return inputs.SelectMany(x => context.Execute(_modules, new[] { x }));
}
}
}
99 changes: 52 additions & 47 deletions Wyam.Core/Modules/WriteFiles.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,65 +57,70 @@ public WriteFiles Where(Func<IDocument, bool> predicate)

public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutionContext context)
{
foreach (IDocument input in inputs.Where(x => _where == null || _where(x)))
foreach (IDocument input in inputs)
{
bool wrote = false;

// WritePath
string path = input.String(MetadataKeys.WritePath);
if (path != null)
{
path = PathHelper.NormalizePath(path);
}

// WriteFileName
if (path == null && input.ContainsKey(MetadataKeys.WriteFileName)
&& input.ContainsKey(MetadataKeys.RelativeFileDir))
if (_where == null || _where(input))
{
path = Path.Combine(input.String(MetadataKeys.RelativeFileDir),
PathHelper.NormalizePath(input.String(MetadataKeys.WriteFileName)));
}
// WritePath
string path = input.String(MetadataKeys.WritePath);
if (path != null)
{
path = PathHelper.NormalizePath(path);
}

// WriteExtension
if (path == null && input.ContainsKey(MetadataKeys.WriteExtension)
&& input.ContainsKey(MetadataKeys.RelativeFilePath))
{
path = Path.ChangeExtension(input.String(MetadataKeys.RelativeFilePath), input.String(MetadataKeys.WriteExtension));
}
// WriteFileName
if (path == null && input.ContainsKey(MetadataKeys.WriteFileName)
&& input.ContainsKey(MetadataKeys.RelativeFileDir))
{
path = Path.Combine(input.String(MetadataKeys.RelativeFileDir),
PathHelper.NormalizePath(input.String(MetadataKeys.WriteFileName)));
}

// Func
if (path == null)
{
path = _path(input);
}
// WriteExtension
if (path == null && input.ContainsKey(MetadataKeys.WriteExtension)
&& input.ContainsKey(MetadataKeys.RelativeFilePath))
{
path = Path.ChangeExtension(input.String(MetadataKeys.RelativeFilePath),
input.String(MetadataKeys.WriteExtension));
}

if (path != null)
{
path = Path.GetFullPath(Path.Combine(context.OutputFolder, path));
if (!string.IsNullOrWhiteSpace(path))
// Func
if (path == null)
{
string pathDirectory = Path.GetDirectoryName(path);
if (!Directory.Exists(pathDirectory))
{
Directory.CreateDirectory(pathDirectory);
}
using (FileStream stream = File.Open(path, FileMode.Create))
path = _path(input);
}

if (path != null)
{
path = Path.GetFullPath(Path.Combine(context.OutputFolder, path));
if (!string.IsNullOrWhiteSpace(path))
{
input.Stream.CopyTo(stream);
string pathDirectory = Path.GetDirectoryName(path);
if (!Directory.Exists(pathDirectory))
{
Directory.CreateDirectory(pathDirectory);
}
using (FileStream stream = File.Open(path, FileMode.Create))
{
input.Stream.CopyTo(stream);
}
context.Trace.Verbose("Wrote file {0}", path);
wrote = true;
yield return input.Clone(new Dictionary<string, object>
{
{MetadataKeys.DestinationFileBase, Path.GetFileNameWithoutExtension(path)},
{MetadataKeys.DestinationFileExt, Path.GetExtension(path)},
{MetadataKeys.DestinationFileName, Path.GetFileName(path)},
{MetadataKeys.DestinationFileDir, Path.GetDirectoryName(path)},
{MetadataKeys.DestinationFilePath, path},
{MetadataKeys.DestinationFilePathBase, PathHelper.RemoveExtension(path)}
});
}
context.Trace.Verbose("Wrote file {0}", path);
wrote = true;
yield return input.Clone(new Dictionary<string, object>
{
{MetadataKeys.DestinationFileBase, Path.GetFileNameWithoutExtension(path)},
{MetadataKeys.DestinationFileExt, Path.GetExtension(path)},
{MetadataKeys.DestinationFileName, Path.GetFileName(path)},
{MetadataKeys.DestinationFileDir, Path.GetDirectoryName(path)},
{MetadataKeys.DestinationFilePath, path },
{MetadataKeys.DestinationFilePathBase, PathHelper.RemoveExtension(path) }
});
}
}

if (!wrote)
{
yield return input;
Expand Down
4 changes: 2 additions & 2 deletions Wyam.Core/Pipelines/ExecutionContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ public ExecutionContext(Engine engine, Pipeline pipeline)
_pipeline = pipeline;
}

public IReadOnlyList<IDocument> Execute(IEnumerable<IModule> modules, IEnumerable<IDocument> inputDocuments)
public IReadOnlyList<IDocument> Execute(IEnumerable<IModule> modules, IEnumerable<IDocument> inputs)
{
// Store the document list before executing the child modules and restore it afterwards
IReadOnlyList<IDocument> documents = _engine.DocumentCollection.Get(_pipeline.Name);
IReadOnlyList<IDocument> results = _pipeline.Execute(modules, inputDocuments);
IReadOnlyList<IDocument> results = _pipeline.Execute(modules, inputs);
_engine.DocumentCollection.Set(_pipeline.Name, documents);
return results;
}
Expand Down
3 changes: 2 additions & 1 deletion Wyam.Core/Wyam.Core.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
Expand Down Expand Up @@ -94,6 +94,7 @@
<Compile Include="Configuration\Configurator.cs" />
<Compile Include="Configuration\IAssemblyCollection.cs" />
<Compile Include="Caching\ExecutionCache.cs" />
<Compile Include="Modules\ForEach.cs" />
<Compile Include="Tracing\DiagnosticsTraceListener.cs" />
<Compile Include="Documents\DocumentCollection.cs" />
<Compile Include="Documents\SeekableStream.cs" />
Expand Down

0 comments on commit 5ccc248

Please sign in to comment.