Merge branch 'master' into colossus-beta

distributor-node/src/services/networking/query-node/schema.graphql

@@ -2029,7 +2029,6 @@ type MemberMetadata implements BaseGraphQLObject {
   member: Membership
   externalResources: [MembershipExternalResource!]
   isVerifiedValidator: Boolean
-  validatorAccount: String
   membercreatedeventmetadata: [MemberCreatedEvent!]
   memberinvitedeventmetadata: [MemberInvitedEvent!]
   memberprofileupdatedeventnewMetadata: [MemberProfileUpdatedEvent!]
@@ -17745,11 +17744,6 @@ input MemberMetadataWhereInput {
   about_in: [String!]
   isVerifiedValidator_eq: Boolean
   isVerifiedValidator_in: [Boolean!]
-  validatorAccount_eq: String
-  validatorAccount_contains: String
-  validatorAccount_startsWith: String
-  validatorAccount_endsWith: String
-  validatorAccount_in: [String!]
   member: MembershipWhereInput
   externalResources_none: MembershipExternalResourceWhereInput
   externalResources_some: MembershipExternalResourceWhereInput
@@ -17783,15 +17777,13 @@ input MemberMetadataCreateInput {
   avatar: JSONObject!
   about: String
   isVerifiedValidator: Boolean
-  validatorAccount: String
 }
 
 input MemberMetadataUpdateInput {
   name: String
   avatar: JSONObject
   about: String
   isVerifiedValidator: Boolean
-  validatorAccount: String
 }
 
 input MembershipGiftedEventWhereInput {
@@ -30510,8 +30502,6 @@ enum MemberMetadataOrderByInput {
   about_DESC
   isVerifiedValidator_ASC
   isVerifiedValidator_DESC
-  validatorAccount_ASC
-  validatorAccount_DESC
 }
 
 enum MemberProfileUpdatedEventOrderByInput {

metadata-protobuf/proto/Membership.proto

@@ -26,10 +26,5 @@ message MembershipMetadata {
     optional ResourceType type = 1;
     optional string value = 2;
   }
-
   repeated ExternalResource externalResources = 5;
-
-  optional string validatorAccount = 6;
 }
-
-

query-node/mappings/src/membership.ts

@@ -163,7 +163,6 @@ async function saveMembershipMetadata(
     avatar,
     externalResources: undefined,
     isVerifiedValidator: false,
-    validatorAccount: metadata?.validatorAccount || undefined,
   })
 
   await store.save<MemberMetadata>(metadataEntity)
@@ -347,13 +346,7 @@ export async function members_MemberProfileUpdated({ store, event }: EventContex
     }
   }
 
-  if (
-    typeof metadata?.validatorAccount === 'string' &&
-    metadata.validatorAccount !== member.metadata.validatorAccount
-  ) {
-    member.metadata.validatorAccount = (metadata.validatorAccount || null) as string | undefined
-    member.metadata.isVerifiedValidator = false
-  }
+  member.metadata.isVerifiedValidator = false
 
   if (newHandle.isSome) {
     setMemberHandle(member, newHandle.unwrap())

query-node/schemas/membership.graphql

@@ -50,7 +50,6 @@ type MemberMetadata @entity {
   externalResources: [MembershipExternalResource] @derivedFrom(field: "memberMetadata")
 
   isVerifiedValidator: Boolean
-  validatorAccount: String
 }
 
 type MembershipEntryPaid @variant {

storage-node/CHANGELOG.md

@@ -1,6 +1,9 @@
 ### 3.9.0
 
 - Add background pruning worker to delete data objects which the node is no longer obligated to store. New optional argument `--cleanup` and `--cleanupInterval`
+- Added new `AcceptPendingObjectsService` that is responsible for periodically sending batch `accept_pending_data_objects` for all the pending data objects. The `POST /files` endpoint now no longer calls the `accept_pending_data_objects` extrinsic for individual uploads, instead, it registers all the pending objects with `AcceptPendingObjectsService`
+- Updated `/state/data` endpoint response headers to return data objects status too i.e. (`pending` or `accepted`)
+- **FIX**: Increase the default timeout value in the `extrinsicWrapper` function to match the transaction validity in the transaction pool
 
 ### 3.8.1
 

storage-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "3.9.0-beta-2",
+  "version": "3.9.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"

storage-node/src/api-spec/openapi.yaml

@@ -281,6 +281,12 @@ components:
         tempDownloads:
           type: integer
           format: int64
+        pendingDirSize:
+          type: integer
+          format: int64
+        pendingObjects:
+          type: integer
+          format: int64
     VersionResponse:
       type: object
       required:

storage-node/src/commands/server.ts

@@ -18,8 +18,9 @@ import {
   MINIMUM_REPLICATION_THRESHOLD,
   performCleanup,
 } from '../services/sync/cleanupService'
+import { AcceptPendingObjectsService } from '../services/sync/acceptPendingObjects'
 import { getStorageBucketIdsByWorkerId } from '../services/sync/storageObligations'
-import { TempDirName, performSync } from '../services/sync/synchronizer'
+import { PendingDirName, performSync, TempDirName } from '../services/sync/synchronizer'
 import { createApp } from '../services/webApi/app'
 import ExitCodes from './../command-base/ExitCodes'
 const fsPromises = fs.promises
@@ -141,6 +142,11 @@ Supported values: warn, error, debug, info. Default:debug`,
       default: 'daily',
       required: false,
     }),
+    maxBatchTxSize: flags.integer({
+      description: 'Maximum number of `accept_pending_data_objects` in a batch transactions.',
+      default: 10,
+      required: false,
+    }),
     ...ApiCommandBase.flags,
   }
 
@@ -212,13 +218,27 @@ Supported values: warn, error, debug, info. Default:debug`,
     await recreateTempDirectory(flags.uploads, TempDirName)
 
     if (fs.existsSync(flags.uploads)) {
-      await loadDataObjectIdCache(flags.uploads, TempDirName)
+      await loadDataObjectIdCache(flags.uploads, TempDirName, PendingDirName)
     }
 
     if (flags.dev) {
       await this.ensureDevelopmentChain()
     }
 
+    const pendingDataObjectsDir = path.join(flags.uploads, PendingDirName)
+
+    const acceptPendingObjectsService = await AcceptPendingObjectsService.create(
+      api,
+      qnApi,
+      workerId,
+      flags.uploads,
+      pendingDataObjectsDir,
+      bucketKeyPairs,
+      writableBuckets,
+      flags.maxBatchTxSize,
+      6000 // Every block
+    )
+
     // Don't run sync job if no buckets selected, to prevent purging
     // any assets.
     if (flags.sync && selectedBuckets.length) {
@@ -279,6 +299,8 @@ Supported values: warn, error, debug, info. Default:debug`,
         maxFileSize,
         uploadsDir: flags.uploads,
         tempFileUploadingDir,
+        pendingDataObjectsDir,
+        acceptPendingObjectsService,
         process: this.config,
         enableUploadingAuth,
         downloadBuckets: selectedBuckets,

storage-node/src/services/caching/localDataObjects.ts

@@ -34,13 +34,20 @@ export async function getDataObjectIDs(): Promise<string[]> {
  * @param uploadDir - uploading directory
  * @param tempDirName - temp directory name
  */
-export async function loadDataObjectIdCache(uploadDir: string, tempDirName: string): Promise<void> {
+export async function loadDataObjectIdCache(
+  uploadDir: string,
+  tempDirName: string,
+  pendingDirName: string
+): Promise<void> {
   await lock.acquireAsync()
 
   const localIds = await getLocalFileNames(uploadDir)
-  // Filter temporary directory name.
+  // Filter temporary & pending directory name.
   const tempDirectoryName = path.parse(tempDirName).name
-  const ids = localIds.filter((dataObjectId) => dataObjectId !== tempDirectoryName)
+  const pendingDirectoryName = path.parse(pendingDirName).name
+  const ids = localIds.filter(
+    (dataObjectId) => dataObjectId !== tempDirectoryName && dataObjectId !== pendingDirectoryName
+  )
 
   ids.forEach((id) => idCache.set(id, 0))
   logger.debug(`Local ID cache loaded.`)

storage-node/src/services/queryNode/api.ts

@@ -27,6 +27,13 @@ import {
   GetDataObjectsConnection,
   GetDataObjectsConnectionQuery,
   GetDataObjectsConnectionQueryVariables,
+  DataObjectsWithBagAndBucketsFragment,
+  // GetDataObjectConnection,
+  // GetDataObjectConnectionQuery,
+  // GetDataObjectConnectionQueryVariables,
+  GetDataObjectsByIds,
+  GetDataObjectsByIdsQuery,
+  GetDataObjectsByIdsQueryVariables,
   GetStorageBucketDetails,
   GetStorageBucketDetailsByWorkerId,
   GetStorageBucketDetailsByWorkerIdQuery,
@@ -347,6 +354,28 @@ export class QueryNodeApi {
     return fullResult
   }
 
+  /**
+   * Returns data objects info by IDs.
+   *
+   * @param bagIds - query filter: data object IDs
+   */
+  public async getDataObjectsByIds(ids: string[]): Promise<Array<DataObjectsWithBagAndBucketsFragment>> {
+    const allIds = [...ids] // Copy to avoid modifying the original array
+    const fullResult: DataObjectsWithBagAndBucketsFragment[] = []
+    while (allIds.length) {
+      const idsBatch = allIds.splice(0, 1000)
+      fullResult.push(
+        ...((await this.multipleEntitiesQuery<GetDataObjectsByIdsQuery, GetDataObjectsByIdsQueryVariables>(
+          GetDataObjectsByIds,
+          { limit: MAX_RESULTS_PER_QUERY, ids: idsBatch },
+          'storageDataObjects'
+        )) || [])
+      )
+    }
+
+    return fullResult
+  }
+
   /**
    * Returns storage bucket IDs.
    *

storage-node/src/services/queryNode/queries/queries.graphql

@@ -138,6 +138,23 @@ query getDataObjectsConnection($dataObjectIds: [ID!], $limit: Int, $cursor: Stri
   }
 }
 
+fragment DataObjectsWithBagAndBuckets on StorageDataObject {
+  id
+  isAccepted
+  storageBag {
+    id
+    storageBuckets {
+      id
+    }
+  }
+}
+
+query getDataObjectsByIds($ids: [ID!], $limit: Int) {
+  storageDataObjects(where: { id_in: $ids }, limit: $limit) {
+    ...DataObjectsWithBagAndBuckets
+  }
+}
+
 fragment QueryNodeStateFields on ProcessorState {
   chainHead
   lastCompleteBlock

storage-node/src/services/queryNode/schema.graphql

@@ -2029,7 +2029,6 @@ type MemberMetadata implements BaseGraphQLObject {
   member: Membership
   externalResources: [MembershipExternalResource!]
   isVerifiedValidator: Boolean
-  validatorAccount: String
   membercreatedeventmetadata: [MemberCreatedEvent!]
   memberinvitedeventmetadata: [MemberInvitedEvent!]
   memberprofileupdatedeventnewMetadata: [MemberProfileUpdatedEvent!]
@@ -17745,11 +17744,6 @@ input MemberMetadataWhereInput {
   about_in: [String!]
   isVerifiedValidator_eq: Boolean
   isVerifiedValidator_in: [Boolean!]
-  validatorAccount_eq: String
-  validatorAccount_contains: String
-  validatorAccount_startsWith: String
-  validatorAccount_endsWith: String
-  validatorAccount_in: [String!]
   member: MembershipWhereInput
   externalResources_none: MembershipExternalResourceWhereInput
   externalResources_some: MembershipExternalResourceWhereInput
@@ -17783,15 +17777,13 @@ input MemberMetadataCreateInput {
   avatar: JSONObject!
   about: String
   isVerifiedValidator: Boolean
-  validatorAccount: String
 }
 
 input MemberMetadataUpdateInput {
   name: String
   avatar: JSONObject
   about: String
   isVerifiedValidator: Boolean
-  validatorAccount: String
 }
 
 input MembershipGiftedEventWhereInput {
@@ -30510,8 +30502,6 @@ enum MemberMetadataOrderByInput {
   about_DESC
   isVerifiedValidator_ASC
   isVerifiedValidator_DESC
-  validatorAccount_ASC
-  validatorAccount_DESC
 }
 
 enum MemberProfileUpdatedEventOrderByInput {