fix(core): build intermediates need to be captured by cache for DTE / parallel builds (#596)

AgentEnder · web-flow · commit cdea76e4b52e · 2023-01-25T18:51:24.000-05:00
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -9,6 +9,8 @@
     <ProjectRelativePath>$([MSBuild]::MakeRelative($(RepoRoot), $(MSBuildProjectDirectory)))</ProjectRelativePath>
     <BaseOutputPath>$(RepoRoot)dist/$(ProjectRelativePath)</BaseOutputPath>
     <OutputPath>$(BaseOutputPath)</OutputPath>
+    <BaseIntermediateOutputPath>$(RepoRoot)dist/intermediates/$(ProjectRelativePath)/obj</BaseIntermediateOutputPath>
+    <IntermediateOutputPath>$(BaseIntermediateOutputPath)</IntermediateOutputPath>
     <AppendTargetFrameworkToOutputPath>true</AppendTargetFrameworkToOutputPath>
   </PropertyGroup>
   <PropertyGroup>
diff --git a/demo/apps/web-frontend/project.json b/demo/apps/web-frontend/project.json
@@ -1,6 +1,6 @@
 {
   "$schema": "../../../node_modules/nx/schemas/project-schema.json",
-  "name": "demo-web-frontend",
+  "name": "demo-frontend",
   "sourceRoot": "demo/apps/web-frontend/src",
   "projectType": "application",
   "targets": {
diff --git a/demo/apps/webapi/Controllers/WeatherForecastController.cs b/demo/apps/webapi/Controllers/WeatherForecastController.cs
@@ -14,7 +14,7 @@ public class WeatherForecastController : ControllerBase
   private readonly ILogger<WeatherForecastController> _logger;
 
   public WeatherForecastController(ILogger<WeatherForecastController> logger)
-  {
+  {
     _logger = logger;
   }
 
diff --git a/demo/apps/webapi/NxDotnet.Test.Webapi.csproj b/demo/apps/webapi/NxDotnet.Test.Webapi.csproj
@@ -1,20 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
-  
   <PropertyGroup>
-    
     <TargetFramework>net6.0</TargetFramework>
-    
     <Nullable>enable</Nullable>
-    
     <ImplicitUsings>enable</ImplicitUsings>
-    
-    <OutputPath>../../../dist/demo/apps/webapi</OutputPath>
   </PropertyGroup>
-  
+
   <ItemGroup>
-    
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.2.3"/>
-    
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.2.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\libs\csharp-models\NxDotnet.Demo.Libs.CsharpModels.csproj" />
   </ItemGroup>
-  
 </Project>
diff --git a/demo/apps/webapi/project.json b/demo/apps/webapi/project.json
@@ -1,12 +1,15 @@
 {
   "$schema": "../../../node_modules/nx/schemas/project-schema.json",
-  "name": "demo-apps-webapi",
+  "name": "demo-webapi",
   "projectType": "application",
   "sourceRoot": "demo/apps/webapi",
   "targets": {
     "build": {
       "executor": "@nx-dotnet/core:build",
-      "outputs": ["{workspaceRoot}/dist/demo/apps/webapi"],
+      "outputs": [
+        "{workspaceRoot}/dist/demo/apps/webapi",
+        "{workspaceRoot}/dist/intermediates/demo/apps/webapi"
+      ],
       "options": {
         "configuration": "Debug",
         "noDependencies": true
diff --git a/demo/libs/csharp-models/Class1.cs b/demo/libs/csharp-models/Class1.cs
@@ -0,0 +1,5 @@
+namespace NxDotnet.Demo.Libs.CsharpModels;
+public class Class1
+{
+
+}
diff --git a/demo/libs/csharp-models/NxDotnet.Demo.Libs.CsharpModels.csproj b/demo/libs/csharp-models/NxDotnet.Demo.Libs.CsharpModels.csproj
@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>
diff --git a/demo/libs/csharp-models/project.json b/demo/libs/csharp-models/project.json
@@ -0,0 +1,28 @@
+{
+  "name": "demo-csharp-models",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "projectType": "library",
+  "sourceRoot": "demo/libs/csharp-models",
+  "targets": {
+    "build": {
+      "executor": "@nx-dotnet/core:build",
+      "outputs": [
+        "{workspaceRoot}/dist/demo/libs/csharp-models",
+        "{workspaceRoot}/dist/intermediates/demo/libs/csharp-models/obj"
+      ],
+      "options": {
+        "configuration": "Debug",
+        "noDependencies": true
+      },
+      "configurations": {
+        "production": {
+          "configuration": "Release"
+        }
+      }
+    },
+    "lint": {
+      "executor": "@nx-dotnet/core:format"
+    }
+  },
+  "tags": []
+}
diff --git a/demo/libs/generated/webapi-swagger/project.json b/demo/libs/generated/webapi-swagger/project.json
@@ -15,5 +15,5 @@
       }
     }
   },
-  "implicitDependencies": ["demo-apps-webapi"]
+  "implicitDependencies": ["demo-webapi"]
 }
diff --git a/nx.json b/nx.json
@@ -63,7 +63,9 @@
       "{workspaceRoot}/tsconfig.base.json",
       "{workspaceRoot}/tslint.json",
       "{workspaceRoot}/nx.json",
-      "{workspaceRoot}/babel.config.json"
+      "{workspaceRoot}/babel.config.json",
+      "{workspaceRoot}/Directory.Build.props",
+      "{workspaceRoot}/Directory.Build.targets"
     ],
     "production": [
       "default",
diff --git a/packages/core/migrations.json b/packages/core/migrations.json
@@ -17,6 +17,12 @@
       "description": "update-1.8.0-beta.0",
       "cli": "nx",
       "implementation": "./src/migrations/1.8.0/remove-output-option"
+    },
+    "update-1.18.1-add-outputs": {
+      "version": "1.18.1-beta.0",
+      "description": "Adds build intermediates to Nx outputs arrays s.t. nx cloud is more effective.",
+      "cli": "nx",
+      "implementation": "./src/migrations/update-1.18.1/add-intermediate-outputs"
     }
   }
 }
diff --git a/packages/core/src/generators/init/templates/root/Directory.Build.props__tmpl__ b/packages/core/src/generators/init/templates/root/Directory.Build.props__tmpl__
@@ -9,6 +9,8 @@
     <ProjectRelativePath>$([MSBuild]::MakeRelative($(RepoRoot), $(MSBuildProjectDirectory)))</ProjectRelativePath>
     <BaseOutputPath>$(RepoRoot)dist/$(ProjectRelativePath)</BaseOutputPath>
     <OutputPath>$(BaseOutputPath)</OutputPath>
+    <BaseIntermediateOutputPath>$(RepoRoot)dist/intermediates/$(ProjectRelativePath)/obj</BaseIntermediateOutputPath>
+    <IntermediateOutputPath>$(BaseIntermediateOutputPath)</IntermediateOutputPath>
     <AppendTargetFrameworkToOutputPath>true</AppendTargetFrameworkToOutputPath>
   </PropertyGroup>
   <PropertyGroup>
diff --git a/packages/core/src/generators/project-reference/generator.spec.ts b/packages/core/src/generators/project-reference/generator.spec.ts
@@ -3,6 +3,10 @@ import { createTreeWithEmptyWorkspace } from '@nrwl/devkit/testing';
 
 import { DotNetClient } from '@nx-dotnet/dotnet';
 
+jest.mock('@nx-dotnet/utils/src/lib/utility-functions/glob', () => ({
+  getProjectFileForNxProject: () => Promise.resolve('my.csproj'),
+}));
+
 import generator from './generator';
 import { NxDotnetGeneratorSchema } from './schema';
 
diff --git a/packages/core/src/generators/project-reference/generator.ts b/packages/core/src/generators/project-reference/generator.ts
@@ -20,7 +20,7 @@ export default async function (
       getProjectFileForNxProject(sourceProject),
     ]);
   } catch {
-    console.warn('Unable to find project files to add dependency!');
+    throw new Error('Unable to find project files to add dependency!');
   }
 
   client.addProjectReference(hostProjectFile, sourceProjectFile);
diff --git a/packages/core/src/generators/project-reference/schema.json b/packages/core/src/generators/project-reference/schema.json
@@ -13,7 +13,8 @@
         "$source": "argv",
         "index": 0
       },
-      "x-prompt": "Which project should the reference be attached to?"
+      "x-prompt": "Which project should the reference be attached to?",
+      "x-dropdown": "projects"
     },
     "reference": {
       "type": "string",
@@ -22,7 +23,8 @@
         "$source": "argv",
         "index": 1
       },
-      "x-prompt": "Which project should the reference point to?"
+      "x-prompt": "Which project should the reference point to?",
+      "x-dropdown": "projects"
     }
   },
   "required": ["project", "reference"]
diff --git a/packages/core/src/generators/utils/generate-project.spec.ts b/packages/core/src/generators/utils/generate-project.spec.ts
@@ -82,9 +82,11 @@ describe('nx-dotnet project generator', () => {
     await GenerateProject(appTree, options, dotnetClient, 'application');
     const config = readProjectConfiguration(appTree, 'test');
     const outputPath = config.targets?.build.outputs || [];
-    expect(outputPath.length).toBe(1);
 
     expect(outputPath[0]).toEqual('{workspaceRoot}/dist/apps/test');
+    expect(outputPath[1]).toEqual(
+      '{workspaceRoot}/dist/intermediates/apps/test',
+    );
   });
 
   it('should include serve target for applications', async () => {
diff --git a/packages/core/src/graph/infer-project.spec.ts b/packages/core/src/graph/infer-project.spec.ts
@@ -52,6 +52,7 @@ describe('infer-project', () => {
         },
         "outputs": Array [
           "{workspaceRoot}/dist/libs/api",
+          "{workspaceRoot}/dist/intermediates/libs/api",
         ],
       }
     `);
diff --git a/packages/core/src/migrations/update-1.18.1/add-intermediate-outputs.ts b/packages/core/src/migrations/update-1.18.1/add-intermediate-outputs.ts
@@ -0,0 +1,132 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import {
+  formatFiles,
+  getProjects,
+  logger,
+  ProjectConfiguration,
+  readNxJson,
+  Tree,
+  updateProjectConfiguration,
+  writeJson,
+} from '@nrwl/devkit';
+import { TargetDefaults } from 'nx/src/config/nx-json';
+import { gt } from 'semver';
+import { XmlDocument } from 'xmldoc';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const nxVersion = require('nx/package.json').version;
+
+export default function update(host: Tree) {
+  const projects = getProjects(host);
+  const directoryBuildPropsExists = host.exists('Directory.Build.props');
+  const directoryBuildPropsUpdated =
+    directoryBuildPropsExists && updateDirectoryBuildProps(host);
+
+  for (const [project, configuration] of projects) {
+    const changed = updateTargetOutputs(
+      directoryBuildPropsUpdated,
+      configuration,
+    );
+    if (changed) {
+      updateProjectConfiguration(host, project, configuration);
+    }
+  }
+
+  updateTargetDefaults(host, directoryBuildPropsUpdated);
+
+  formatFiles(host);
+}
+
+function updateDirectoryBuildProps(host: Tree): boolean {
+  const contents = host.read('Directory.Build.props', 'utf-8');
+  if (!contents) {
+    logger.warn('Unable to read "Directory.Build.props"');
+    return false;
+  }
+  const xml = new XmlDocument(contents);
+  const propertyGroups = xml.childrenNamed('PropertyGroup');
+  const outputManipulationGroup = propertyGroups.find((group) =>
+    group.childNamed('OutputPath'),
+  );
+  if (!outputManipulationGroup) {
+    logger.warn(
+      'Unable to find property group containing output manipulation in Directory.Build.props',
+    );
+    return false;
+  }
+  outputManipulationGroup.children.push(
+    new XmlDocument(
+      `<BaseIntermediateOutputPath>$(RepoRoot)dist/intermediates/$(ProjectRelativePath)/obj</BaseIntermediateOutputPath>`,
+    ),
+  );
+  outputManipulationGroup.children.push(
+    new XmlDocument(
+      `<IntermediateOutputPath>$(BaseIntermediateOutputPath)</IntermediateOutputPath>`,
+    ),
+  );
+  host.write('Directory.Build.props', xml.toString());
+  return true;
+}
+
+function updateTargetOutputs(
+  directoryBuildPropsUpdated: boolean,
+  configuration: ProjectConfiguration,
+): boolean {
+  let changed = false;
+
+  const targets = Object.values(configuration.targets ?? {}).filter(
+    (x) => x.executor === '@nx-dotnet/core:build',
+  );
+
+  for (const target of targets) {
+    if (directoryBuildPropsUpdated) {
+      if (!target.outputs?.some((x) => x.includes('intermediates'))) {
+        const prefix = gt(nxVersion, '15.0.0-beta.0') ? '{workspaceRoot}/' : '';
+        target.outputs?.push(
+          prefix + `dist/intermediates/${configuration.root}`,
+        );
+        changed = true;
+      }
+    } else {
+      if (!target.outputs?.some((x) => x.includes('obj'))) {
+        const prefix = gt(nxVersion, '15.0.0-beta.0')
+          ? '{projectRoot}/'
+          : `${configuration.root}/`;
+        target.outputs?.push(prefix + `obj`);
+        changed = true;
+      }
+    }
+  }
+  return changed;
+}
+
+function updateTargetDefaults(host: Tree, directoryBuildPropsUpdated: boolean) {
+  let changed = false;
+  const nxJson = readNxJson();
+  const targetDefaults: TargetDefaults | undefined = nxJson.targetDefaults;
+
+  if (!targetDefaults) {
+    return;
+  }
+
+  const configuration = targetDefaults['@nx-dotnet/core:build'];
+  if (configuration) {
+    if (directoryBuildPropsUpdated) {
+      if (!configuration.outputs?.some((x) => x.includes('intermediates'))) {
+        configuration.outputs?.push(
+          `{workspaceRoot}/dist/intermediates/{projectRoot}`,
+        );
+        changed = true;
+      }
+    } else {
+      if (!configuration.outputs?.some((x) => x.includes('obj'))) {
+        configuration.outputs?.push(`{projectRoot}/obj`);
+        changed = true;
+      }
+    }
+  }
+
+  if (changed) {
+    writeJson(host, 'nx.json', nxJson);
+  }
+}
diff --git a/packages/core/src/models/build-executor-configuration.ts b/packages/core/src/models/build-executor-configuration.ts
@@ -11,13 +11,16 @@ export function GetBuildExecutorConfiguration(
   // eslint-disable-next-line @typescript-eslint/no-var-requires
   const nxVersion = require('nx/package.json').version;
 
-  const outputDirectory =
-    (lt(nxVersion, '15.0.0-beta.0') ? '' : '{workspaceRoot}/') +
-    `dist/${projectRoot}`;
+  const outputs = lt(nxVersion, '15.0.0-beta.0')
+    ? [`dist/${projectRoot}`, `dist/intermediates/${projectRoot}`]
+    : [
+        `{workspaceRoot}/dist/${projectRoot}`,
+        `{workspaceRoot}/dist/intermediates/${projectRoot}`,
+      ];
 
   return {
     executor: '@nx-dotnet/core:build',
-    outputs: [outputDirectory],
+    outputs,
     options: {
       configuration: 'Debug',
       noDependencies: true,

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"$schema": "../../../node_modules/nx/schemas/project-schema.json",`
`3`		`- "name": "demo-web-frontend",`
	`3`	`+ "name": "demo-frontend",`
`4`	`4`	`"sourceRoot": "demo/apps/web-frontend/src",`
`5`	`5`	`"projectType": "application",`
`6`	`6`	`"targets": {`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ public class WeatherForecastController : ControllerBase`
`14`	`14`	`private readonly ILogger<WeatherForecastController> _logger;`
`15`	`15`
`16`	`16`	`public WeatherForecastController(ILogger<WeatherForecastController> logger)`
`17`		`- {`
	`17`	`+ {`
`18`	`18`	`_logger = logger;`
`19`	`19`	`}`
`20`	`20`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +namespace NxDotnet.Demo.Libs.CsharpModels;
 +public class Class1
 +{
++
 +}
Original file line number	Diff line number	Diff line change
`@@ -15,5 +15,5 @@`
`15`	`15`	`}`
`16`	`16`	`}`
`17`	`17`	`},`
`18`		`- "implicitDependencies": ["demo-apps-webapi"]`
	`18`	`+ "implicitDependencies": ["demo-webapi"]`
`19`	`19`	`}`