feat!: add data, cache and temp dirs to FileSystem (#1306)

Signed-off-by: Ariel Gentile <gentilester@gmail.com>

BREAKING CHANGE:

Agent-produced files will now be divided in different system paths depending on their nature: data, temp and cache. Previously, they were located at a single location, defaulting to a temporary directory.

If you specified a custom path in `FileSystem` object constructor, you now must provide an object containing `baseDataPath`, `baseTempPath` and `baseCachePath`. They can point to the same path, although it's recommended to specify different path to avoid future file clashes.

packages/anoncreds/src/utils/tails.ts

@@ -2,11 +2,11 @@ import type { AgentContext, FileSystem } from '@aries-framework/core'
 
 import { TypedArrayEncoder, InjectionSymbols } from '@aries-framework/core'
 
-const getTailsFilePath = (basePath: string, tailsHash: string) => `${basePath}/afj/anoncreds/tails/${tailsHash}`
+const getTailsFilePath = (cachePath: string, tailsHash: string) => `${cachePath}/anoncreds/tails/${tailsHash}`
 
 export function tailsFileExists(agentContext: AgentContext, tailsHash: string): Promise<boolean> {
   const fileSystem = agentContext.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-  const tailsFilePath = getTailsFilePath(fileSystem.basePath, tailsHash)
+  const tailsFilePath = getTailsFilePath(fileSystem.cachePath, tailsHash)
 
   return fileSystem.exists(tailsFilePath)
 }
@@ -27,7 +27,7 @@ export async function downloadTailsFile(
 
     // hash is used as file identifier
     const tailsExists = await tailsFileExists(agentContext, tailsHashBase58)
-    const tailsFilePath = getTailsFilePath(fileSystem.basePath, tailsHashBase58)
+    const tailsFilePath = getTailsFilePath(fileSystem.cachePath, tailsHashBase58)
     agentContext.config.logger.debug(
       `Tails file for ${tailsLocation} ${tailsExists ? 'is stored' : 'is not stored'} at ${tailsFilePath}`
     )

packages/askar/src/utils/askarWalletConfig.ts

@@ -18,7 +18,16 @@ export const keyDerivationMethodToStoreKeyMethod = (keyDerivationMethod?: KeyDer
   return correspondenceTable[keyDerivationMethod] as StoreKeyMethod
 }
 
-export const uriFromWalletConfig = (walletConfig: WalletConfig, basePath: string): { uri: string; path?: string } => {
+/**
+ * Creates a proper askar wallet URI value based on walletConfig
+ * @param walletConfig WalletConfig object
+ * @param afjDataPath framework data path (used in case walletConfig.storage.path is undefined)
+ * @returns string containing the askar wallet URI
+ */
+export const uriFromWalletConfig = (
+  walletConfig: WalletConfig,
+  afjDataPath: string
+): { uri: string; path?: string } => {
   let uri = ''
   let path
 
@@ -31,7 +40,7 @@ export const uriFromWalletConfig = (walletConfig: WalletConfig, basePath: string
     if (walletConfig.storage.inMemory) {
       uri = 'sqlite://:memory:'
     } else {
-      path = `${(walletConfig.storage.path as string) ?? basePath + '/wallet'}/${walletConfig.id}/sqlite.db`
+      path = (walletConfig.storage.path as string) ?? `${afjDataPath}/wallet/${walletConfig.id}/sqlite.db`
       uri = `sqlite://${path}`
     }
   } else if (walletConfig.storage.type === 'postgres') {

packages/askar/src/wallet/AskarWallet.ts

@@ -289,7 +289,7 @@ export class AskarWallet implements Wallet {
     }
 
     try {
-      const { uri } = uriFromWalletConfig(this.walletConfig, this.fileSystem.basePath)
+      const { uri } = uriFromWalletConfig(this.walletConfig, this.fileSystem.dataPath)
       await Store.remove(uri)
     } catch (error) {
       const errorMessage = `Error deleting wallet '${this.walletConfig.id}': ${error.message}`
@@ -689,7 +689,7 @@ export class AskarWallet implements Wallet {
   }
 
   private async getAskarWalletConfig(walletConfig: WalletConfig) {
-    const { uri, path } = uriFromWalletConfig(walletConfig, this.fileSystem.basePath)
+    const { uri, path } = uriFromWalletConfig(walletConfig, this.fileSystem.dataPath)
 
     // Make sure path exists before creating the wallet
     if (path) {

packages/core/src/modules/indy/services/IndyUtilitiesService.ts

@@ -63,7 +63,7 @@ export class IndyUtilitiesService {
   public async downloadTails(hash: string, tailsLocation: string): Promise<BlobReaderHandle> {
     try {
       this.logger.debug(`Checking to see if tails file for URL ${tailsLocation} has been stored in the FileSystem`)
-      const filePath = `${this.fileSystem.basePath}/afj/tails/${hash}`
+      const filePath = `${this.fileSystem.cachePath}/tails/${hash}`
 
       const tailsExists = await this.fileSystem.exists(filePath)
       this.logger.debug(`Tails file for ${tailsLocation} ${tailsExists ? 'is stored' : 'is not stored'} at ${filePath}`)

packages/core/src/modules/ledger/IndyPool.ts

@@ -181,7 +181,7 @@ export class IndyPool {
     if (this.poolConfig.genesisPath) return this.poolConfig.genesisPath
 
     // Determine the genesisPath
-    const genesisPath = this.fileSystem.basePath + `/afj/genesis-${this.poolConfig.id}.txn`
+    const genesisPath = this.fileSystem.tempPath + `/genesis-${this.poolConfig.id}.txn`
     // Store genesis data if provided
     if (this.poolConfig.genesisTransactions) {
       await this.fileSystem.write(genesisPath, this.poolConfig.genesisTransactions)

packages/core/src/storage/FileSystem.ts

@@ -5,11 +5,14 @@ export interface DownloadToFileOptions {
 }
 
 export interface FileSystem {
-  readonly basePath: string
+  readonly dataPath: string
+  readonly cachePath: string
+  readonly tempPath: string
 
   exists(path: string): Promise<boolean>
   createDirectory(path: string): Promise<void>
   write(path: string, data: string): Promise<void>
   read(path: string): Promise<string>
+  delete(path: string): Promise<void>
   downloadToFile(url: string, path: string, options?: DownloadToFileOptions): Promise<void>
 }

packages/core/src/storage/migration/UpdateAssistant.ts

@@ -157,13 +157,18 @@ export class UpdateAssistant<Agent extends BaseAgent<any> = BaseAgent> {
             `Successfully updated agent storage from version ${update.fromVersion} to version ${update.toVersion}`
           )
         }
+        // Delete backup file, as it is not needed anymore
+        await this.fileSystem.delete(this.getBackupPath(updateIdentifier))
       } catch (error) {
         this.agent.config.logger.fatal('An error occurred while updating the wallet. Restoring backup', {
           error,
         })
         // In the case of an error we want to restore the backup
         await this.restoreBackup(updateIdentifier)
 
+        // Delete backup file, as wallet was already restored (backup-error file will persist though)
+        await this.fileSystem.delete(this.getBackupPath(updateIdentifier))
+
         throw error
       }
     } catch (error) {
@@ -192,7 +197,7 @@ export class UpdateAssistant<Agent extends BaseAgent<any> = BaseAgent> {
   }
 
   private getBackupPath(backupIdentifier: string) {
-    return `${this.fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
+    return `${this.fileSystem.dataPath}/migration/backup/${backupIdentifier}`
   }
 
   private async createBackup(backupIdentifier: string) {

packages/core/src/storage/migration/__tests__/0.1.test.ts

@@ -48,8 +48,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
         dependencyManager
       )
 
-      const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
       const updateAssistant = new UpdateAssistant(agent, {
         v0_1ToV0_2: {
           mediationRoleUpdateStrategy,
@@ -79,10 +77,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
       delete storageService.records.MEDIATOR_ROUTING_RECORD
       expect(storageService.records).toMatchSnapshot(mediationRoleUpdateStrategy)
 
-      // Need to remove backupFiles after each run so we don't get IOErrors
-      const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-      unlinkSync(backupPath)
-
       await agent.shutdown()
       await agent.wallet.delete()
     }
@@ -110,8 +104,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
       dependencyManager
     )
 
-    const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
     const updateAssistant = new UpdateAssistant(agent, {
       v0_1ToV0_2: {
         mediationRoleUpdateStrategy: 'doNotChange',
@@ -142,10 +134,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
     delete storageService.records.MEDIATOR_ROUTING_RECORD
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 
@@ -174,8 +162,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
       dependencyManager
     )
 
-    const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
     const updateAssistant = new UpdateAssistant(agent, {
       v0_1ToV0_2: {
         mediationRoleUpdateStrategy: 'doNotChange',
@@ -206,10 +192,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
     delete storageService.records.MEDIATOR_ROUTING_RECORD
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 
@@ -242,8 +224,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
       dependencyManager
     )
 
-    const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
     const updateAssistant = new UpdateAssistant(agent, {
       v0_1ToV0_2: {
         mediationRoleUpdateStrategy: 'doNotChange',
@@ -274,10 +254,6 @@ describe('UpdateAssistant | v0.1 - v0.2', () => {
     delete storageService.records.MEDIATOR_ROUTING_RECORD
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 

packages/core/src/storage/migration/__tests__/0.2.test.ts

@@ -13,7 +13,6 @@ import { UpdateAssistant } from '../UpdateAssistant'
 
 const backupDate = new Date('2022-01-21T22:50:20.522Z')
 jest.useFakeTimers().setSystemTime(backupDate)
-const backupIdentifier = backupDate.getTime()
 
 const walletConfig = {
   id: `Wallet: 0.2 Update`,
@@ -46,8 +45,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
       dependencyManager
     )
 
-    const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
     const updateAssistant = new UpdateAssistant(agent, {
       v0_1ToV0_2: {
         mediationRoleUpdateStrategy: 'doNotChange',
@@ -83,10 +80,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
     delete storageService.records.MEDIATOR_ROUTING_RECORD
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 
@@ -119,8 +112,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
       dependencyManager
     )
 
-    const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
     // We need to manually initialize the wallet as we're using the in memory wallet service
     // When we call agent.initialize() it will create the wallet and store the current framework
     // version in the in memory storage service. We need to manually set the records between initializing
@@ -137,10 +128,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
     delete storageService.records.MEDIATOR_ROUTING_RECORD
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 
@@ -170,8 +157,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
       dependencyManager
     )
 
-    const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-
     // We need to manually initialize the wallet as we're using the in memory wallet service
     // When we call agent.initialize() it will create the wallet and store the current framework
     // version in the in memory storage service. We need to manually set the records between initializing
@@ -189,10 +174,6 @@ describe('UpdateAssistant | v0.2 - v0.3.1', () => {
 
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 

packages/core/src/storage/migration/__tests__/0.3.test.ts

@@ -75,10 +75,6 @@ describe('UpdateAssistant | v0.3 - v0.3.1', () => {
     delete storageService.records.MEDIATOR_ROUTING_RECORD
     expect(storageService.records).toMatchSnapshot()
 
-    // Need to remove backupFiles after each run so we don't get IOErrors
-    const backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
-    unlinkSync(backupPath)
-
     await agent.shutdown()
     await agent.wallet.delete()
 

packages/core/src/storage/migration/__tests__/backup.test.ts

@@ -32,7 +32,7 @@ describe('UpdateAssistant | Backup', () => {
   beforeEach(async () => {
     agent = new Agent(agentOptions)
     const fileSystem = agent.dependencyManager.resolve<FileSystem>(InjectionSymbols.FileSystem)
-    backupPath = `${fileSystem.basePath}/afj/migration/backup/${backupIdentifier}`
+    backupPath = `${fileSystem.dataPath}/migration/backup/${backupIdentifier}`
 
     // If tests fail it's possible the cleanup has been skipped. So remove before running tests
     const doesFileSystemExist = await fileSystem.exists(backupPath)
@@ -85,11 +85,16 @@ describe('UpdateAssistant | Backup', () => {
     // Backup should not exist before update
     expect(await fileSystem.exists(backupPath)).toBe(false)
 
+    const walletSpy = jest.spyOn(agent.wallet, 'export')
+
     // Create update
     await updateAssistant.update()
 
-    // Backup should exist after update
-    expect(await fileSystem.exists(backupPath)).toBe(true)
+    // A wallet export should have been initiated
+    expect(walletSpy).toHaveBeenCalledWith({ key: agent.wallet.walletConfig?.key, path: backupPath })
+
+    // Backup should be cleaned after update
+    expect(await fileSystem.exists(backupPath)).toBe(false)
 
     expect(
       (await credentialRepository.getAll(agent.context)).sort((a, b) => a.id.localeCompare(b.id))
@@ -142,8 +147,8 @@ describe('UpdateAssistant | Backup', () => {
 
     expect(updateError?.cause?.message).toEqual("Uh oh I'm broken")
 
-    // Backup should exist after update
-    expect(await fileSystem.exists(backupPath)).toBe(true)
+    // Only backup error should exist after update
+    expect(await fileSystem.exists(backupPath)).toBe(false)
     expect(await fileSystem.exists(`${backupPath}-error`)).toBe(true)
 
     // Wallet should be same as when we started because of backup

packages/indy-sdk/src/ledger/IndySdkPool.ts

@@ -180,7 +180,7 @@ export class IndySdkPool {
     if (this.poolConfig.genesisPath) return this.poolConfig.genesisPath
 
     // Determine the genesisPath
-    const genesisPath = this.fileSystem.basePath + `/afj/genesis-${this.poolConfig.id}.txn`
+    const genesisPath = this.fileSystem.tempPath + `/genesis-${this.poolConfig.id}.txn`
     // Store genesis data if provided
     if (this.poolConfig.genesisTransactions) {
       await this.fileSystem.write(genesisPath, this.poolConfig.genesisTransactions)