Added the PostProcessHasDependencies flag, still need to add unit tests

RELEASE.md

@@ -1,5 +1,6 @@
 # 1.0.0-beta.65
 
+- Added a new pipeline `PostProcessHasDependencies` property that indicates the post-process phase of a pipeline should depend on the post-process phase(s) of the pipeline dependencies. This is helpful in certain situations where you need a pipeline to run after other post-process phases from dependencies.
 - Added a `GenerateRedirects.AlwaysCreateAdditionalOutput()` method to allow creating additional redirect files even if no redirects are specified (I.e. if redirects are also being generated from another source).
 - Added additional overloads to `GenerateRedirects.WithAdditionalOutput()` that can accept the execution context in the delegate and/or return a `Task`.
 

src/core/Statiq.Common/Execution/IPipeline.cs

@@ -44,7 +44,10 @@ public interface IPipeline : IReadOnlyPipeline
         /// <inheritdoc/>
         new bool Deployment { get; set; }
 
+        /// <inheritdoc/>
+        new bool PostProcessHasDependencies { get; set; }
+
         /// <inheritdoc/>
         new ExecutionPolicy ExecutionPolicy { get; set; }
     }
-}
+}

src/core/Statiq.Common/Execution/IReadOnlyPipeline.cs

@@ -31,9 +31,28 @@ public interface IReadOnlyPipeline
         /// </summary>
         bool Deployment { get; }
 
+        /// <summary>
+        /// Indicates that the post-process phase of this pipeline should have
+        /// dependencies on the post-process phase(s) of it's dependencies.
+        /// </summary>
+        /// <remarks>
+        /// Normally all post-process phases are executed concurrently after
+        /// all process phases have completed. This also means that a post-process
+        /// phase can access documents from the process phase of any pipeline (other
+        /// than those with <see cref="Isolated"/> set to <c>true</c>) but not
+        /// from the post-process phase in other pipelines. Sometimes it can be
+        /// helpful for the post-process phase of a pipeline to itself have a
+        /// dependency on the post-process phase of it's pipeline dependencies.
+        /// Setting this to <c>true</c> will cause the post-process phase of this
+        /// pipeline to wait to execute until all post-process phases of it's
+        /// dependencies have completed, and will allow it to access documents
+        /// from those post-process phases.
+        /// </remarks>
+        bool PostProcessHasDependencies { get; }
+
         /// <summary>
         /// Indicates when the pipeline is executed.
         /// </summary>
         ExecutionPolicy ExecutionPolicy { get; }
     }
-}
+}

src/core/Statiq.Common/Execution/NamedPipelineWrapper.cs

@@ -43,6 +43,12 @@ public bool Deployment
             set => Pipeline.Deployment = value;
         }
 
+        public bool PostProcessHasDependencies
+        {
+            get => Pipeline.PostProcessHasDependencies;
+            set => Pipeline.PostProcessHasDependencies = value;
+        }
+
         public ExecutionPolicy ExecutionPolicy
         {
             get => Pipeline.ExecutionPolicy;

src/core/Statiq.Common/IO/NormalizedPath.cs

@@ -1059,7 +1059,7 @@ public static string ReplaceInvalidPathChars(string path, char newChar = '-')
         public static bool ReplaceInvalidPathChars(in Span<char> path, char newChar = '-') =>
             path.Replace(Path.GetInvalidPathChars(), newChar);
 
-        public const string OptimizeFileNameReservedChars = "_~:/\\?#[]@!$&'()*+;=};,";
+        public const string OptimizeFileNameReservedChars = "_~:/\\?#[]@!$&'()*+={};,";
 
         public static string OptimizeFileName(
             string fileName,

src/core/Statiq.Core/Documents/PhaseOutputs.cs

@@ -85,10 +85,10 @@ public IEnumerator<IDocument> GetEnumerator()
             // Only add from same deployment value pipelines (if a deployment pipeline, non-deployment phases were added above)
             if (_currentPhase.Phase == Phase.Process)
             {
-                HashSet<string> transitivePipelineDependencies = GatherProcessPhasePipelineDependencies(_currentPhase);
+                HashSet<string> transitiveProcessDependencies = GatherPipelineDependencies(_currentPhase, Phase.Process);
                 outputs.AddRange(_phaseResults
                     .Where(x => _pipelines[x.Key].Deployment == _currentPhase.Pipeline.Deployment)
-                    .Where(x => transitivePipelineDependencies.Contains(x.Key))
+                    .Where(x => transitiveProcessDependencies.Contains(x.Key))
                     .Select(x => KeyValuePair.Create(x.Key, GetOutputs(x.Value, Phase.Process))));
             }
 
@@ -99,6 +99,17 @@ public IEnumerator<IDocument> GetEnumerator()
                 outputs.AddRange(_phaseResults
                     .Where(x => _pipelines[x.Key].Deployment == _currentPhase.Pipeline.Deployment)
                     .Select(x => KeyValuePair.Create(x.Key, GetOutputs(x.Value, Phase.Process))));
+
+                // Also add other post-process phases if the post-process dependencies flag is set
+                // Need to AddOrReplace existing outputs since we've already added some for process phases
+                if (_currentPhase.Pipeline.PostProcessHasDependencies)
+                {
+                    HashSet<string> transitivePostProcessDependencies = GatherPipelineDependencies(_currentPhase, Phase.PostProcess);
+                    outputs.AddOrReplaceRange(_phaseResults
+                        .Where(x => _pipelines[x.Key].Deployment == _currentPhase.Pipeline.Deployment)
+                        .Where(x => transitivePostProcessDependencies.Contains(x.Key))
+                        .Select(x => KeyValuePair.Create(x.Key, GetOutputs(x.Value, Phase.PostProcess))));
+                }
             }
 
             // If we're in the output phase non-deployment outputs were added above
@@ -119,16 +130,16 @@ private static DocumentList<IDocument> GetOutputs(PhaseResult[] phaseResults, Ph
             return p == -1 ? DocumentList<IDocument>.Empty : phaseResults[p].Outputs.ToDocumentList();
         }
 
-        private HashSet<string> GatherProcessPhasePipelineDependencies(PipelinePhase phase, HashSet<string> transientDependencies = null)
+        private HashSet<string> GatherPipelineDependencies(
+            PipelinePhase currentPhase,
+            Phase dependentPhase,
+            HashSet<string> transientDependencies = null)
         {
-            if (transientDependencies is null)
-            {
-                transientDependencies = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-            }
-            foreach (PipelinePhase dependency in phase.Dependencies.Where(x => x.Phase == Phase.Process))
+            transientDependencies ??= new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            foreach (PipelinePhase dependency in currentPhase.Dependencies.Where(x => x.Phase == dependentPhase))
             {
                 transientDependencies.Add(dependency.PipelineName);
-                GatherProcessPhasePipelineDependencies(dependency, transientDependencies);
+                GatherPipelineDependencies(dependency, dependentPhase, transientDependencies);
             }
             return transientDependencies;
         }
@@ -184,7 +195,7 @@ private void ValidatePipeline(string pipelineName)
                 {
                     throw new InvalidOperationException($"Cannot access documents from currently executing pipeline {pipelineName} while in the {nameof(Phase.Process)} phase");
                 }
-                if (!GatherProcessPhasePipelineDependencies(_currentPhase).Contains(pipelineName))
+                if (!GatherPipelineDependencies(_currentPhase, Phase.Process).Contains(pipelineName))
                 {
                     throw new InvalidOperationException($"Cannot access documents from pipeline {pipelineName} without a dependency while in the {nameof(Phase.Process)} phase");
                 }
@@ -209,4 +220,4 @@ private void ValidateCurrent()
             }
         }
     }
-}
+}

src/core/Statiq.Core/Execution/Engine.cs

@@ -723,12 +723,14 @@ internal static PipelinePhase[] GetPipelinePhases(IPipelineCollection pipelines,
             // Make a pass through non-isolated post-process phases to set dependencies to all non-isolated process phases
             foreach (PipelinePhases pipelinePhases in phasesByPipeline.Values.Where(x => !x.Pipeline.Isolated))
             {
-                // We only need to add phases from other pipelines with same deployment setting since deployment pipeline input
-                // phases are dependent on non-deloyment pipeline output phases below (I.e. deployment pipelines won't execute *any*
-                // phases until all non-deployment pipelines are completely done)
+                // We only need to add process phases from other pipelines with same deployment setting
                 pipelinePhases.PostProcess.Dependencies =
                     pipelinePhases.PostProcess.Dependencies
-                        .Concat(phasesByPipeline.Values.Where(x => x != pipelinePhases && !x.Pipeline.Isolated && pipelinePhases.Pipeline.Deployment == x.Pipeline.Deployment).Select(x => x.Process))
+                        .Concat(phasesByPipeline.Values
+                            .Where(x => x != pipelinePhases // ...it's not the same pipeline
+                                && !x.Pipeline.Isolated // ...and it's not isolated
+                                && pipelinePhases.Pipeline.Deployment == x.Pipeline.Deployment) // ...and it's the same deployment setting
+                            .Select(x => x.Process))
                         .ToArray();
             }
 
@@ -783,6 +785,7 @@ PipelinePhases Visit(string name, IPipeline pipeline)
                 {
                     // Visit dependencies if this isn't an isolated pipeline
                     List<PipelinePhase> processDependencies = new List<PipelinePhase>();
+                    List<PipelinePhase> postProcessDependencies = new List<PipelinePhase>();
                     foreach (string dependencyName in pipeline.GetAllDependencies(pipelines))
                     {
                         if (!pipelines.TryGetValue(dependencyName, out IPipeline dependency))
@@ -800,20 +803,30 @@ PipelinePhases Visit(string name, IPipeline pipeline)
                             throw new PipelineException($"Non-deployment pipeline {name} cannot have a dependency on deployment pipeline {dependencyName}");
                         }
 
-                        processDependencies.Add(Visit(dependencyName, dependency).Process);
+                        // Add process phase dependencies to the process phase of all dependencies
+                        // and add post-process phase dependencies to the post-process phase of all dependencies if
+                        // the post-process dependency flag is set
+                        PipelinePhases dependencyPhases = Visit(dependencyName, dependency);
+                        processDependencies.Add(dependencyPhases.Process);
+                        if (pipeline.PostProcessHasDependencies)
+                        {
+                            postProcessDependencies.Add(dependencyPhases.PostProcess);
+                        }
                     }
 
                     // Add the phases (by this time all dependencies should have been added)
                     pipelinePhases = new PipelinePhases(pipeline);
 
                     pipelinePhases.Input = new PipelinePhase(pipeline, name, Phase.Input, pipeline.InputModules, logger);
 
-                    // Makes sure the process phase is also dependent on it's input phase
+                    // Makes sure the process phase is also dependent on it's own pipeline's input phase
                     processDependencies.Insert(0, pipelinePhases.Input);
                     pipelinePhases.Process = new PipelinePhase(pipeline, name, Phase.Process, pipeline.ProcessModules, logger, processDependencies.ToArray());
 
-                    // Post-process dependencies will be added after all pipelines have been processed
-                    pipelinePhases.PostProcess = new PipelinePhase(pipeline, name, Phase.PostProcess, pipeline.PostProcessModules, logger, pipelinePhases.Process);
+                    // Post-process dependencies to all process phases will be added after all pipelines have been processed
+                    // Make sure the post-process phase is also dependent on it's own pipeline's process phase
+                    postProcessDependencies.Insert(0, pipelinePhases.Process);
+                    pipelinePhases.PostProcess = new PipelinePhase(pipeline, name, Phase.PostProcess, pipeline.PostProcessModules, logger, postProcessDependencies.ToArray());
 
                     // The output phase is dependent on it's post-process phase
                     pipelinePhases.Output = new PipelinePhase(pipeline, name, Phase.Output, pipeline.OutputModules, logger, pipelinePhases.PostProcess);

src/core/Statiq.Core/Execution/ExecutionPipeline.cs

@@ -72,6 +72,9 @@ protected ExecutionPipeline()
         /// <inheritdoc/>
         public virtual bool Deployment { get; set; }
 
+        /// <inheritdoc/>
+        public virtual bool PostProcessHasDependencies { get; set; }
+
         /// <inheritdoc/>
         public virtual ExecutionPolicy ExecutionPolicy { get; set; }
 
@@ -102,4 +105,4 @@ context.Phase switch
         protected virtual Task<IEnumerable<IDocument>> ExecuteOutputPhaseAsync(IExecutionContext context) =>
             throw new NotImplementedException();
     }
-}
+}

src/core/Statiq.Core/Execution/Pipeline.cs

@@ -58,6 +58,9 @@ public ModuleList OutputModules
         /// <inheritdoc/>
         public bool Deployment { get; set; }
 
+        /// <inheritdoc/>
+        public bool PostProcessHasDependencies { get; set; }
+
         /// <inheritdoc/>
         public ExecutionPolicy ExecutionPolicy { get; set; }
 
@@ -67,4 +70,4 @@ public ModuleList OutputModules
         /// <inheritdoc/>
         IReadOnlyCollection<string> IReadOnlyPipeline.DependencyOf => DependencyOf;
     }
-}
+}

src/core/Statiq.Testing/Execution/TestPipeline.cs

@@ -29,6 +29,8 @@ public class TestPipeline : IPipeline
 
         public bool Deployment { get; set; }
 
+        public bool PostProcessHasDependencies { get; set; }
+
         public ExecutionPolicy ExecutionPolicy { get; set; }
 
         public TestPipeline(params IModule[] processModules)
@@ -41,4 +43,4 @@ public TestPipeline(IEnumerable<IModule> processModules)
             ProcessModules.AddRange(processModules);
         }
     }
-}
+}