Merge pull request #4925 from kdembler/argus-tweaks

[argus] few tweaks
Joystream · Oct 16, 2023 · 8d3d5b4 · 8d3d5b4
2 parents 85cc60c + 74fc629
commit 8d3d5b4
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 3 deletions.
diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md
@@ -1,8 +1,14 @@
 ##
 
+### 1.4.0
+
+- Add 1s delay between fetching pages for QN state sync. This will allow QN to have some breathing space and process other requests that may have been stuck because of heavy processing.
+- Add `Access-Control-Expose-Headers` response header to allow web clients checking cache status of downloaded file.
+- Include response headers in `http` logs
+
 ### 1.3.1
 
-- Batch fetching of objects details from query-node [#4921](https://github.com/Joystream/joystream/pull/4921)
+- **FIX** QN state sync: The QN state sync that runs on startup and on interval, has been split to multiple paginated queries so that it doesn't crash QN's GraphQL server because of huge payload: [#4921](https://github.com/Joystream/joystream/pull/4921)
 
 ### 1.3.0
 

diff --git a/distributor-node/package.json b/distributor-node/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"

diff --git a/distributor-node/src/services/httpApi/HttpApiBase.ts b/distributor-node/src/services/httpApi/HttpApiBase.ts
@@ -61,7 +61,10 @@ export abstract class HttpApiBase {
       winstonInstance: this.logger,
       level: 'http',
       dynamicMeta: (req, res) => {
-        return { prematurelyClosed: res.locals.prematurelyClosed ?? false }
+        return {
+          prematurelyClosed: res.locals.prematurelyClosed ?? false,
+          res: { headers: res.getHeaders(), statusCode: res.statusCode },
+        }
       },
     }
   }

diff --git a/distributor-node/src/services/httpApi/controllers/public.ts b/distributor-node/src/services/httpApi/controllers/public.ts
@@ -224,6 +224,7 @@ export class PublicApiController {
     res.setHeader('timing-allow-origin', '*')
     res.setHeader('accept-ranges', 'bytes')
     res.setHeader('content-disposition', 'inline')
+    res.setHeader('access-control-expose-headers', 'x-cache, x-data-source')
 
     switch (objectStatus.type) {
       case ObjectStatusType.Available:
@@ -271,6 +272,7 @@ export class PublicApiController {
     })
 
     res.setHeader('timing-allow-origin', '*')
+    res.setHeader('access-control-expose-headers', 'x-cache, x-data-source')
 
     switch (objectStatus.type) {
       case ObjectStatusType.Available:

diff --git a/distributor-node/src/services/networking/query-node/api.ts b/distributor-node/src/services/networking/query-node/api.ts
@@ -230,6 +230,8 @@ export class QueryNodeApi {
           GetDataObjectsWithBagsByIdsQueryVariables
         >(GetDataObjectsWithBagsByIds, { bagIds: bagIdsBatch, limit: bagIdsBatch.length }, 'storageBags'))
       )
+      // wait 1s between requests to avoid overloading the query node
+      await new Promise((resolve) => setTimeout(resolve, 1000))
     }
 
     return fullResult.map((bag) => bag.objects).flat()