Merge pull request #5024 from mnaamani/colossus-move-instead-of-rename

storage-node: moveFile

storage-node/src/services/helpers/moveFile.ts

@@ -0,0 +1,21 @@
+import fs from 'fs'
+const fsPromises = fs.promises
+
+/**
+ * Asynchronously copies `src` to `dest`. If `dest` already exists, operation is aborted
+ * to avoid corrupting destination file or removal by nodejs when an error occurs after
+ * it has been opened for writing.
+ * Use this function instead of `rename` to move files between volumes, otherwise
+ * operation fails with: EXDEV: cross-device link not permitted.
+ * @param src A path to the source file.
+ * @param dest A path to the destination file.
+ */
+export async function moveFile(src: fs.PathLike, dest: fs.PathLike): Promise<void> {
+  // `COPYFILE_EXCL` flag causes the copy operation to fail if `dest` already exists.
+  const { COPYFILE_EXCL } = fs.constants
+  // Try to copy source file to destination
+  await fsPromises.copyFile(src, dest, COPYFILE_EXCL)
+
+  // Only if copy operation succeeds we delete the source file
+  await fsPromises.unlink(src)
+}

storage-node/src/services/sync/acceptPendingObjects.ts

@@ -9,6 +9,7 @@ import logger from '../logger'
 import { QueryNodeApi } from '../queryNode/api'
 import { acceptPendingDataObjectsBatch } from '../runtime/extrinsics'
 import { hashFile } from '../helpers/hashing'
+import { moveFile } from '../helpers/moveFile'
 
 const fsPromises = fs.promises
 
@@ -159,7 +160,7 @@ export class AcceptPendingObjectsService {
         await fsPromises.unlink(currentPath)
       } catch {
         // If the file does not exist in the uploads directory, proceed with the rename
-        await fsPromises.rename(currentPath, newPath)
+        await moveFile(currentPath, newPath)
         registerNewDataObjectId(dataObjectId)
         addDataObjectIdToCache(dataObjectId)
       }

storage-node/src/services/sync/tasks.ts

@@ -14,6 +14,7 @@ import {
 } from '../caching/localDataObjects'
 import { isNewDataObject } from '../caching/newUploads'
 import { hashFile } from '../helpers/hashing'
+import { moveFile } from '../helpers/moveFile'
 const fsPromises = fs.promises
 
 /**
@@ -147,7 +148,7 @@ export class DownloadFileTask implements SyncTask {
       })
       await streamPipeline(request, fileStream)
       await this.verifyDownloadedFile(tempFilePath)
-      await fsPromises.rename(tempFilePath, filepath)
+      await moveFile(tempFilePath, filepath)
       addDataObjectIdToCache(this.dataObjectId)
     } catch (err) {
       logger.warn(`Sync - fetching data error for ${url}: ${err}`, { err })

storage-node/src/services/webApi/controllers/filesApi.ts

@@ -25,6 +25,7 @@ import {
 } from '../types'
 import { AppConfig, WebApiError, getHttpStatusCodeByError, sendResponseWithError } from './common'
 import _ from 'lodash'
+import { moveFile } from '../../helpers/moveFile'
 const fsPromises = fs.promises
 
 /**
@@ -143,7 +144,7 @@ export async function uploadFile(
     const { pendingDataObjectsDir } = res.locals
     const newPathPending = path.join(pendingDataObjectsDir, dataObjectId)
 
-    await fsPromises.rename(fileObj.path, newPathPending)
+    await moveFile(fileObj.path, newPathPending)
 
     res.status(201).json({
       id: hash,