colossus: v3.8.2 backport fixes

Joystream · Jan 4, 2024 · 8cf54d5 · 8cf54d5
1 parent 8897fc5
commit 8cf54d5
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 4 deletions.
diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md
@@ -1,3 +1,7 @@
+### 3.8.2
+
+- Hotfix - backporting fixes
+
 ### 3.8.1
 
 - Hotfix: Fix call stack size exceeded when handling large number of initial object to sync.

diff --git a/storage-node/package.json b/storage-node/package.json
@@ -1,7 +1,7 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "3.8.1",
+  "version": "3.8.2",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"

diff --git a/storage-node/src/services/queryNode/api.ts b/storage-node/src/services/queryNode/api.ts
@@ -298,17 +298,18 @@ export class QueryNodeApi {
    */
   public async getDataObjectDetails(bagIds: string[]): Promise<Array<DataObjectDetailsFragment>> {
     const allBagIds = [...bagIds] // Copy to avoid modifying the original array
-    const fullResult: DataObjectDetailsFragment[] = []
+    let fullResult: DataObjectDetailsFragment[] = []
     while (allBagIds.length) {
       const bagIdsBatch = allBagIds.splice(0, 1000)
       const input: StorageBagWhereInput = { id_in: bagIdsBatch }
-      fullResult.push(
+      fullResult = [
+        ...fullResult,
         ...(await this.multipleEntitiesWithPagination<
           DataObjectDetailsFragment,
           GetDataObjectConnectionQuery,
           GetDataObjectConnectionQueryVariables
         >(GetDataObjectConnection, { limit: MAX_RESULTS_PER_QUERY, bagIds: input }, 'storageDataObjectsConnection'))
-      )
+      ]
     }
 
     return fullResult

diff --git a/storage-node/src/services/sync/tasks.ts b/storage-node/src/services/sync/tasks.ts
@@ -109,6 +109,17 @@ export class DownloadFileTask implements SyncTask {
         if (!res.ok) {
           logger.error(`Sync - unexpected status code(${res.statusCode}) for ${res?.request?.url}`)
         }
+
+        // Handle 'error' event on Response too, because it will be emitted if request was
+        // prematurely aborted/closed due to timeout and the response still was not completed
+        // See: https://github.com/nodejs/node/blob/cd171576b2d1376dae3eb371b6da5ccf04dc4a85/lib/_http_client.js#L439-L441
+        res.on('error', (err: Error) => {
+          logger.error(`Sync - fetching data error for ${this.url}: ${err}`, { err })
+        })
+      })
+
+      request.on('error', (err) => {
+        logger.error(`Sync - fetching data error for ${this.url}: ${err}`, { err })
       })
       await streamPipeline(request, fileStream)
       await this.verifyDownloadedFile(tempFilePath)