implement links (loose dependencies) for build optimizations

src/Terrabuild.Configuration.Tests/Project.fs

@@ -15,6 +15,7 @@ let parseProject() =
     let expectedProject =
         let project =
             { Dependencies = Set [ "../../libraries/shell-lib" ] |> Some
+              Links = None
               Outputs = Set [ "dist" ] |> Some
               Ignores = None
               Includes = None
@@ -80,6 +81,7 @@ let parseProject2() =
     let expectedProject =
         let project =
             { Dependencies = None
+              Links = None
               Outputs = None
               Ignores = None
               Includes = None

src/Terrabuild.Configuration/Gen/ProjectParser.fsi

@@ -2,6 +2,7 @@
 module internal Terrabuild.Configuration.Project.Parser
 type token = 
   | DEPENDENCIES
+  | LINKS
   | OUTPUTS
   | IGNORES
   | INCLUDES
@@ -50,6 +51,7 @@ type token =
   | FALSE
 type tokenId = 
     | TOKEN_DEPENDENCIES
+    | TOKEN_LINKS
     | TOKEN_OUTPUTS
     | TOKEN_IGNORES
     | TOKEN_INCLUDES
@@ -111,6 +113,7 @@ type nonTerminalId =
     | NONTERM_Project
     | NONTERM_ProjectComponents
     | NONTERM_ProjectDependencies
+    | NONTERM_ProjectLinks
     | NONTERM_ProjectOutputs
     | NONTERM_ProjectIgnores
     | NONTERM_ProjectIncludes

src/Terrabuild.Configuration/ProjectParser/AST.fs

@@ -6,6 +6,7 @@ open Errors
 [<RequireQualifiedAccess>]
 type ProjectComponents =
     | Dependencies of string list
+    | Links of string list
     | Outputs of string list
     | Ignores of string list
     | Includes of string list
@@ -14,6 +15,7 @@ type ProjectComponents =
 type Project = {
     Init: string option
     Dependencies: Set<string> option
+    Links: Set<string> option
     Outputs: Set<string> option
     Ignores: Set<string> option
     Includes: Set<string> option
@@ -27,6 +29,12 @@ with
             | [value] -> value |> Set.ofList |> Some
             | _ -> TerrabuildException.Raise("multiple dependencies declared")
 
+        let links =
+            match components |> List.choose (function | ProjectComponents.Links value -> Some value | _ -> None) with
+            | [] -> None
+            | [value] -> value |> Set.ofList |> Some
+            | _ -> TerrabuildException.Raise("multiple links declared")
+
         let outputs =
             match components |> List.choose (function | ProjectComponents.Outputs value -> Some value | _ -> None) with
             | [] -> None
@@ -53,6 +61,7 @@ with
 
         { Init = init
           Dependencies = dependencies
+          Links = links
           Outputs = outputs
           Ignores = ignores
           Includes = includes

src/Terrabuild.Configuration/ProjectParser/Lexer.fsl

@@ -51,6 +51,7 @@ rule token = parse
     | "project" { PROJECT }
     | "target" { TARGET }
     | "dependencies" { DEPENDENCIES }
+    | "links" { LINKS }
     | "outputs" { OUTPUTS }
     | "ignores" { IGNORES }
     | "includes" { INCLUDES }

src/Terrabuild.Configuration/ProjectParser/Parser.fsy

@@ -33,7 +33,7 @@ let debugPrint s = ignore s
 %token TRIM UPPER LOWER VERSION DOUBLE_QUESTION QUESTION COLON
 %token EOF
 %token PROJECT EXTENSION TARGET 
-%token DEPENDENCIES OUTPUTS IGNORES INCLUDES LABELS VARIABLES CONTAINER INIT SCRIPT DEPENDS_ON REBUILD DEFAULTS NAME
+%token DEPENDENCIES LINKS OUTPUTS IGNORES INCLUDES LABELS VARIABLES CONTAINER INIT SCRIPT DEPENDS_ON REBUILD DEFAULTS NAME
 
 // associativity and precedences
 
@@ -86,12 +86,15 @@ Project:
 ProjectComponents:
     | /* empty */ { [] }
     | ProjectComponents ProjectDependencies { $1 @ [$2] }
+    | ProjectComponents ProjectLinks { $1 @ [$2] }
     | ProjectComponents ProjectOutputs { $1 @ [$2] }
     | ProjectComponents ProjectIgnores { $1 @ [$2] }
     | ProjectComponents ProjectIncludes { $1 @ [$2] }
     | ProjectComponents ProjectLabels { $1 @ [$2] }
 ProjectDependencies:
     | DEPENDENCIES EQUAL ListOfString { ProjectComponents.Dependencies $3 }
+ProjectLinks:
+    | LINKS EQUAL ListOfString { ProjectComponents.Links $3 }
 ProjectOutputs:
     | OUTPUTS EQUAL ListOfString { ProjectComponents.Outputs $3 }
 ProjectIgnores:

src/Terrabuild.Extensibility/Extensions.fs

@@ -13,13 +13,15 @@ type ProjectInfo = {
     Outputs: Set<string>
     Ignores: Set<string>
     Dependencies: Set<string>
+    Links: Set<string>
     Includes: Set<string>
 }
 with
     static member Default = {
         Outputs = Set.empty
         Ignores = Set.empty
         Dependencies = Set.empty
+        Links = Set.empty
         Includes = Set [ "**/*" ]
     }
 

src/Terrabuild/Core/Configuration.fs

@@ -94,6 +94,7 @@ type private LazyScript = Lazy<Terrabuild.Scripting.Script>
 [<RequireQualifiedAccess>]
 type private LoadedProject = {
     Dependencies: string set
+    Links: string set
     Includes: string set
     Ignores: string set
     Outputs: string set
@@ -227,6 +228,7 @@ let read (options: Options) =
             with Ignores = projectInfo.Ignores + (projectConfig.Project.Ignores |> Option.defaultValue Set.empty)
                  Outputs = projectInfo.Outputs + (projectConfig.Project.Outputs |> Option.defaultValue Set.empty)
                  Dependencies = projectInfo.Dependencies + (projectConfig.Project.Dependencies |> Option.defaultValue Set.empty)
+                 Links = projectInfo.Links + (projectConfig.Project.Links |> Option.defaultValue Set.empty)
                  Includes = projectInfo.Includes + (projectConfig.Project.Includes |> Option.defaultValue Set.empty) }
 
         let labels = projectConfig.Project.Labels
@@ -237,6 +239,9 @@ let read (options: Options) =
         let projectDependencies =
             projectInfo.Dependencies
             |> Set.map (fun dep -> FS.workspaceRelative options.Workspace projectDir dep)
+        let projectLinks =
+            projectInfo.Links
+            |> Set.map (fun dep -> FS.workspaceRelative options.Workspace projectDir dep)
 
         let projectTargets = projectConfig.Targets
 
@@ -247,6 +252,7 @@ let read (options: Options) =
             |> Set.union projectInfo.Includes
 
         { LoadedProject.Dependencies = projectDependencies
+          LoadedProject.Links = projectLinks
           LoadedProject.Includes = includes
           LoadedProject.Ignores = projectIgnores
           LoadedProject.Outputs = projectOutputs
@@ -257,28 +263,37 @@ let read (options: Options) =
 
 
     // this is the final stage: create targets and create the project
-    let finalizeProject projectId (projectDef: LoadedProject) (projectDependencies: Map<string, Project>) =
+    let finalizeProject projectId (projectDef: LoadedProject) (projectReferences: Map<string, Project>) =
         let projectDir = projectId
 
         // get dependencies on files
         let files =
             projectDir
             |> IO.enumerateFilesBut (projectDef.Includes) (projectDef.Outputs + projectDef.Ignores)
             |> Set
+
         let filesHash =
             files
             |> Seq.sort
             |> Hash.sha256files
 
-        let versions =
-            projectDependencies
-            |> Map.map (fun _ depProj -> depProj.Hash)
-
         let dependenciesHash =
-            versions.Values
+            let projectDependencies =
+                projectReferences
+                |> Map.filter (fun projectId _ -> projectDef.Dependencies |> Set.contains projectId)
+
+            let versionDependencies =
+                projectDependencies
+                |> Map.map (fun _ depProj -> depProj.Hash)
+
+            versionDependencies.Values
             |> Seq.sort
             |> Hash.sha256strings
 
+        let versions = 
+            projectReferences
+            |> Map.map (fun _ depProj -> depProj.Hash)
+
         // NOTE: this is the hash (modulo target name) used for reconcialiation across executions
         let projectHash =
             [ projectId; filesHash; dependenciesHash ]
@@ -447,7 +462,7 @@ let read (options: Options) =
 
             // await dependencies to be loaded
             let awaitedProjects =
-                loadedProject.Dependencies
+                (loadedProject.Dependencies + loadedProject.Links)
                 |> Seq.map (fun awaitedProjectId -> hub.GetComputed<Project> awaitedProjectId)
                 |> Array.ofSeq