Suspend incomplete requests

packages/core/core/src/RequestTracker.js

@@ -54,6 +54,7 @@ import {report} from './ReporterRunner';
 import {PromiseQueue} from '@parcel/utils';
 import type {Cache} from '@parcel/cache';
 import {getConfigKeyContentHash} from './requests/ConfigRequest';
+import {getFeatureFlag} from '@parcel/feature-flags';
 
 export const requestGraphEdgeTypes = {
   subrequest: 2,
@@ -132,6 +133,7 @@ type Request<TInput, TResult> = {|
   id: string,
   +type: RequestType,
   input: TInput,
+  incompleteRequest?: boolean,
   run: ({|input: TInput, ...StaticRunOpts<TResult>|}) => Async<TResult>,
 |};
 
@@ -1029,6 +1031,10 @@ export default class RequestTracker {
   options: ParcelOptions;
   signal: ?AbortSignal;
   stats: Map<RequestType, number> = new Map();
+  incompleteRequests: Map<
+    Request<mixed, mixed>['id'],
+    {|deferred: Deferred<void>, promise: Promise<void>, timerId: TimeoutID|},
+  > = new Map();
 
   constructor({
     graph,
@@ -1162,11 +1168,63 @@ export default class RequestTracker {
     this.graph.replaceSubrequests(requestNodeId, subrequestContextKeys);
   }
 
+  flushCachedRequest(requestId: ContentKey) {
+    let existingPromise = this.incompleteRequests.get(requestId);
+
+    if (existingPromise) {
+      logger.verbose({
+        origin: '@parcel/core',
+        message: 'Incomplete request promise resolved',
+        meta: {trackableEvent: 'incomplete_promise_resolved'},
+      });
+
+      clearTimeout(existingPromise.timerId);
+      existingPromise.deferred.resolve();
+    }
+  }
+
+  waitForFullRequest(requestId: ContentKey): Promise<void> {
+    let existingPromise = this.incompleteRequests.get(requestId);
+
+    if (existingPromise) {
+      return existingPromise.promise;
+    }
+
+    let deferredPromise = makeDeferredWithPromise();
+
+    let timerId = setTimeout(() => {
+      logger.verbose({
+        origin: '@parcel/core',
+        message: 'Incomplete request promise timed out',
+        meta: {trackableEvent: 'incomplete_promise_timeout'},
+      });
+      deferredPromise.deferred.resolve();
+    }, 5_000);
+
+    this.incompleteRequests.set(requestId, {...deferredPromise, timerId});
+
+    return deferredPromise.promise;
+  }
+
   async runRequest<TInput, TResult>(
     request: Request<TInput, TResult>,
     opts?: ?RunRequestOpts,
   ): Promise<TResult> {
-    let requestId = this.graph.hasContentKey(request.id)
+    let hasKey = this.graph.hasContentKey(request.id);
+
+    if (
+      request.incompleteRequest &&
+      !hasKey &&
+      getFeatureFlag('devDepRequestBugFix')
+    ) {
+      // To avoid possible race conditions with the worker farm build cache, we
+      // suspend processing of an incomplete request until the full details are back
+      // from another worker.
+      await this.waitForFullRequest(request.id);
+      hasKey = this.graph.hasContentKey(request.id);
+    }
+
+    let requestId = hasKey
       ? this.graph.getNodeIdByContentKey(request.id)
       : undefined;
     let hasValidResult = requestId != null && this.hasValidResult(requestId);
@@ -1228,6 +1286,9 @@ export default class RequestTracker {
       deferred.resolve(false);
       throw err;
     } finally {
+      if (getFeatureFlag('devDepRequestBugFix')) {
+        this.flushCachedRequest(request.id);
+      }
       this.graph.replaceSubrequests(requestNodeId, [...subRequestContentKeys]);
     }
   }

packages/core/core/src/requests/AssetGraphRequest.js

@@ -67,6 +67,7 @@ type AssetGraphRequest = {|
   +type: typeof requestTypes.asset_graph_request,
   run: RunInput => Async<AssetGraphRequestResult>,
   input: AssetGraphRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 export default function createAssetGraphRequest(

packages/core/core/src/requests/AssetRequest.js

@@ -32,6 +32,7 @@ export type AssetRequest = {|
   +type: typeof requestTypes.asset_request,
   run: (RunInput<AssetRequestResult>) => Async<AssetRequestResult>,
   input: AssetRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 export default function createAssetRequest(

packages/core/core/src/requests/BundleGraphRequest.js

@@ -82,6 +82,7 @@ type BundleGraphRequest = {|
   +type: typeof requestTypes.bundle_graph_request,
   run: RunInput => Async<BundleGraphResult>,
   input: BundleGraphRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 export default function createBundleGraphRequest(

packages/core/core/src/requests/DevDepRequest.js

@@ -190,14 +190,22 @@ export async function runDevDepRequest<TResult>(
   await api.runRequest<null, DevDepRequestResult | void>({
     id: 'dev_dep_request:' + devDepRequest.specifier + ':' + devDepRequest.hash,
     type: requestTypes.dev_dep_request,
+    incompleteRequest: !(
+      devDepRequest.invalidateOnFileChange &&
+      devDepRequest.invalidateOnFileCreate
+    ),
     run: ({api}) => {
-      for (let filePath of nullthrows(devDepRequest.invalidateOnFileChange)) {
+      for (let filePath of nullthrows(
+        devDepRequest.invalidateOnFileChange,
+        'DevDepRequest missing invalidateOnFileChange',
+      )) {
         api.invalidateOnFileUpdate(filePath);
         api.invalidateOnFileDelete(filePath);
       }
 
       for (let invalidation of nullthrows(
         devDepRequest.invalidateOnFileCreate,
+        'DevDepRequest missing invalidateOnFileCreate',
       )) {
         api.invalidateOnFileCreate(invalidation);
       }

packages/core/core/src/requests/EntryRequest.js

@@ -36,6 +36,7 @@ export type EntryRequest = {|
   +type: typeof requestTypes.entry_request,
   run: (RunOpts<EntryResult>) => Async<EntryResult>,
   input: ProjectPath,
+  incompleteRequest?: boolean,
 |};
 
 export type EntryResult = {|

packages/core/core/src/requests/PackageRequest.js

@@ -34,6 +34,7 @@ export type PackageRequest = {|
   +type: typeof requestTypes.package_request,
   run: (RunInput<BundleInfo>) => Async<BundleInfo>,
   input: PackageRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 export function createPackageRequest(

packages/core/core/src/requests/ParcelBuildRequest.js

@@ -46,6 +46,7 @@ export type ParcelBuildRequest = {|
   +type: typeof requestTypes.parcel_build_request,
   run: (RunInput<ParcelBuildRequestResult>) => Async<ParcelBuildRequestResult>,
   input: ParcelBuildRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 export default function createParcelBuildRequest(

packages/core/core/src/requests/ParcelConfigRequest.js

@@ -56,6 +56,7 @@ export type ParcelConfigRequest = {|
   type: typeof requestTypes.parcel_config_request,
   input: null,
   run: (RunOpts<ConfigAndCachePath>) => Async<ConfigAndCachePath>,
+  incompleteRequest?: boolean,
 |};
 
 type ParcelConfigChain = {|

packages/core/core/src/requests/PathRequest.js

@@ -58,6 +58,7 @@ export type PathRequest = {|
   +type: typeof requestTypes.path_request,
   run: (RunOpts<?AssetGroup>) => Async<?AssetGroup>,
   input: PathRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 export type PathRequestInput = {|

packages/core/core/src/requests/TargetRequest.js

@@ -93,6 +93,7 @@ export type TargetRequest = {|
   +type: typeof requestTypes.target_request,
   run: (RunOpts<Array<Target>>) => Async<Array<Target>>,
   input: Entry,
+  incompleteRequest?: boolean,
 |};
 
 const type = 'target_request';

packages/core/core/src/requests/ValidationRequest.js

@@ -17,6 +17,7 @@ type ValidationRequest = {|
   +type: typeof requestTypes.validation_request,
   run: (RunOpts<void>) => Async<void>,
   input: ValidationRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 type RunOpts<TResult> = {|

packages/core/core/src/requests/WriteBundleRequest.js

@@ -61,6 +61,7 @@ export type WriteBundleRequest = {|
   +type: typeof requestTypes.write_bundle_request,
   run: (RunInput<PackagedBundleInfo>) => Async<PackagedBundleInfo>,
   input: WriteBundleRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 /**

packages/core/core/src/requests/WriteBundlesRequest.js

@@ -33,6 +33,7 @@ export type WriteBundlesRequest = {|
     RunInput<Map<string, PackagedBundleInfo>>,
   ) => Async<Map<string, PackagedBundleInfo>>,
   input: WriteBundlesRequestInput,
+  incompleteRequest?: boolean,
 |};
 
 /**

packages/core/core/test/test-utils.js

@@ -53,6 +53,7 @@ export const DEFAULT_OPTIONS: ParcelOptions = {
   },
   featureFlags: {
     exampleFeature: false,
+    devDepRequestBugFix: false,
     configKeyInvalidation: false,
   },
 };

packages/core/feature-flags/src/index.js

@@ -8,6 +8,7 @@ export type FeatureFlags = _FeatureFlags;
 export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
   exampleFeature: false,
   configKeyInvalidation: false,
+  devDepRequestBugFix: false,
 };
 
 let featureFlagValues: FeatureFlags = {...DEFAULT_FEATURE_FLAGS};

packages/core/feature-flags/src/types.js

@@ -9,4 +9,8 @@ export type FeatureFlags = {|
    * `config.getConfigFrom(..., {packageKey: '...'})` and the value itself hasn't changed.
    */
   +configKeyInvalidation: boolean,
+  /**
+   * Enable experimental fix for the DevDepRequest build cache bug
+   */
+  +devDepRequestBugFix: boolean,
 |};

packages/core/integration-tests/test/cache.js

@@ -1318,6 +1318,7 @@ describe('cache', function () {
         let b = await testCache({
           featureFlags: {
             exampleFeature: false,
+            devDepRequestBugFix: false,
             configKeyInvalidation: true,
           },
           async setup() {
@@ -1378,6 +1379,7 @@ describe('cache', function () {
         let b = await testCache({
           featureFlags: {
             exampleFeature: false,
+            devDepRequestBugFix: false,
             configKeyInvalidation: true,
           },
           async setup() {
@@ -1438,6 +1440,7 @@ describe('cache', function () {
         let b = await testCache({
           featureFlags: {
             exampleFeature: false,
+            devDepRequestBugFix: false,
             configKeyInvalidation: true,
           },
           async setup() {