Skip to content

Commit

Permalink
[eas-update] Increase asset upload timeout to 90s and reset on upload…
Browse files Browse the repository at this point in the history
… retry for slow connections
  • Loading branch information
wschurman committed Oct 11, 2023
1 parent 49b6e1f commit 644f57c
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 9 deletions.
18 changes: 12 additions & 6 deletions packages/eas-cli/src/commands/update/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -250,20 +250,22 @@ export default class UpdatePublish extends EasCommand {
realizedPlatforms = Object.keys(assets) as PublishPlatform[];

// Timeout mechanism:
// - Start with 60 second timeout. 60 seconds is chosen because the cloud function that processes
// uploaded assets has a timeout of 60 seconds.
// - Each time one or more assets reports as ready, reset the timeout to 60 seconds.
// - Start upload. Internally, uploadAssetsAsync uploads them all and then checks for successful
// - Start with NO_ACTIVITY_TIMEOUT. 90 seconds is chosen because the cloud function that processes
// uploaded assets has a timeout of 60 seconds and uploading can take some time on a slow connection.
// - Each time one or more assets reports as ready, reset the timeout.
// - Each time an asset upload begins, reset the timeout. This includes retries.
// - Start upload. Internally, uploadAssetsAsync uploads them all first and then checks for successful
// processing every (5 + n) seconds with a linear backoff of n + 1 second.
// - At the same time as upload is started, start timeout checker which checks every 1 second to see
// if timeout has been reached. When timeout expires, send a cancellation signal to currently running
// upload function call to instruct it to stop uploading or checking for successful processing.
const NO_ACTIVITY_TIMEOUT = 90 * 1000; // 90 seconds
let lastUploadedStorageKeys = new Set<string>();
let lastAssetUploadResults: {
asset: RawAsset & { storageKey: string };
finished: boolean;
}[] = [];
let timeAtWhichToTimeout = Date.now() + 60 * 1000; // sixty seconds from now
let timeAtWhichToTimeout = Date.now() + NO_ACTIVITY_TIMEOUT;
const cancelationToken = { isCanceledOrFinished: false };

const uploadResults = await Promise.race([
Expand All @@ -277,14 +279,18 @@ export default class UpdatePublish extends EasCommand {
assetUploadResults.filter(r => r.finished).map(r => r.asset.storageKey)
);
if (!areSetsEqual(currentUploadedStorageKeys, lastUploadedStorageKeys)) {
timeAtWhichToTimeout = Date.now() + 60 * 1000; // reset timeout to sixty seconds from now
timeAtWhichToTimeout = Date.now() + NO_ACTIVITY_TIMEOUT; // reset timeout to NO_ACTIVITY_TIMEOUT
lastUploadedStorageKeys = currentUploadedStorageKeys;
lastAssetUploadResults = assetUploadResults;
}

const totalAssets = assetUploadResults.length;
const missingAssetCount = assetUploadResults.filter(a => !a.finished).length;
assetSpinner.text = `Uploading (${totalAssets - missingAssetCount}/${totalAssets})`;
},
() => {
// when an upload is retried, reset the timeout as we know this will now need more time
timeAtWhichToTimeout = Date.now() + NO_ACTIVITY_TIMEOUT; // reset timeout to NO_ACTIVITY_TIMEOUT
}
),
(async () => {
Expand Down
9 changes: 7 additions & 2 deletions packages/eas-cli/src/project/publish.ts
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,8 @@ export async function uploadAssetsAsync(
cancelationToken: { isCanceledOrFinished: boolean },
onAssetUploadResultsChanged: (
assetUploadResults: { asset: RawAsset & { storageKey: string }; finished: boolean }[]
) => void
) => void,
onAssetUploadBegin: () => void
): Promise<AssetUploadResult> {
let assets: RawAsset[] = [];
let platform: keyof CollectedAssets;
Expand Down Expand Up @@ -486,7 +487,11 @@ export async function uploadAssetsAsync(
throw Error('Canceled upload');
}
const presignedPost: PresignedPost = JSON.parse(specifications[i]);
await uploadWithPresignedPostWithRetryAsync(missingAsset.path, presignedPost);
await uploadWithPresignedPostWithRetryAsync(
missingAsset.path,
presignedPost,
onAssetUploadBegin
);
});
}),
]);
Expand Down
4 changes: 3 additions & 1 deletion packages/eas-cli/src/uploads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,15 @@ export async function uploadFileAtPathToGCSAsync(

export async function uploadWithPresignedPostWithRetryAsync(
file: string,
presignedPost: PresignedPost
presignedPost: PresignedPost,
onAssetUploadBegin: () => void
): Promise<Response> {
return await promiseRetry(
async retry => {
// retry fetch errors (usually connection or DNS errors)
let response: Response;
try {
onAssetUploadBegin();
response = await uploadWithPresignedPostAsync(file, presignedPost);
} catch (e: any) {
return retry(e);
Expand Down

0 comments on commit 644f57c

Please sign in to comment.