Allow disk-based persistence to be completely disabled

.vscode/launch.json

@@ -17,6 +17,19 @@
       "skipFiles": ["node_modules/*/**", "<node_internals>/*/**"],
       "outputCapture": "std"
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Azurite Service - Loki, in-memory",
+      "cwd": "${workspaceFolder}",
+      "runtimeArgs": ["-r", "ts-node/register"],
+      "args": ["${workspaceFolder}/src/azurite.ts", "-d", "debug.log", "--inMemoryPersistence"],
+      "env": {
+        "AZURITE_ACCOUNTS": ""
+      },
+      "skipFiles": ["node_modules/*/**", "<node_internals>/*/**"],
+      "outputCapture": "std"
+    },
     {
       "type": "node",
       "request": "launch",

ChangeLog.md

@@ -4,6 +4,10 @@
 
 ## Upcoming Release
 
+General:
+
+- Add `--inMemoryStorage` option and related configs to persist all data in-memory without disk persistence. (issue #2227)
+
 Blob:
 
 - Fix validation of Blob SAS token when using the second key for an account in `AZURITE_ACCOUNTS`

README.mcr.md

@@ -74,6 +74,8 @@ Above command will try to start Azurite image with configurations:
 
 `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host.
 
+`--inMemoryPersistence` disable persisting any data to disk. If the Azurite process is terminated, all data is lost.
+
 > If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters.
 
 > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows.

README.md

@@ -198,6 +198,7 @@ Following extension configurations are supported:
 - `azurite.oauth` OAuth oauthentication level. Candidate level values: `basic`.
 - `azurite.skipApiVersionCheck` Skip the request API version check, by default false.
 - `azurite.disableProductStyleUrl` Force parsing storage account name from request Uri path, instead of from request Uri host.
+- `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost.
 
 ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite)
 
@@ -257,6 +258,8 @@ Above command will try to start Azurite image with configurations:
 
 `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host.
 
+`--inMemoryPersistence` disable persisting any data to disk. If the Azurite process is terminated, all data is lost.
+
 > If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters.
 
 > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows.
@@ -430,6 +433,16 @@ Optional. When using FQDN instead of IP in request Uri host, by default Azurite
 --disableProductStyleUrl
 ```
 
+### Use in-memory storage
+
+Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost.
+By default, LokiJS persists blob and queue metadata to disk and content to extent files. Table storage
+persists all data to disk. This behavior can be disabled using this option.
+
+```cmd
+--inMemoryStorage
+```
+
 ### Command Line Options Differences between Azurite V2
 
 Azurite V3 supports SharedKey, Account Shared Access Signature (SAS), Service SAS, OAuth, and Public Container Access authentications, you can use any Azure Storage SDKs or tools like Storage Explorer to connect Azurite V3 with any authentication strategy.

src/azurite.ts

@@ -98,7 +98,8 @@ async function main() {
     env.key(),
     env.pwd(),
     env.oauth(),
-    env.disableProductStyleUrl()
+    env.disableProductStyleUrl(),
+    env.inMemoryPersistence(),
   );
 
   const tableConfig = new TableConfiguration(
@@ -115,7 +116,8 @@ async function main() {
     env.key(),
     env.pwd(),
     env.oauth(),
-    env.disableProductStyleUrl()
+    env.disableProductStyleUrl(),
+    env.inMemoryPersistence(),
   );
 
   // We use logger singleton as global debugger logger to track detailed outputs cross layers

src/blob/BlobConfiguration.ts

@@ -39,7 +39,8 @@ export default class BlobConfiguration extends ConfigurationBase {
     key: string = "",
     pwd: string = "",
     oauth?: string,
-    disableProductStyleUrl: boolean = false
+    disableProductStyleUrl: boolean = false,
+    isMemoryPersistence: boolean = false,
   ) {
     super(
       host,
@@ -54,7 +55,8 @@ export default class BlobConfiguration extends ConfigurationBase {
       key,
       pwd,
       oauth,
-      disableProductStyleUrl
+      disableProductStyleUrl,
+      isMemoryPersistence,
     );
   }
 }

src/blob/BlobEnvironment.ts

@@ -40,6 +40,10 @@ if (!(args as any).config.name) {
     .option(["", "oauth"], 'Optional. OAuth level. Candidate values: "basic"')
     .option(["", "cert"], "Optional. Path to certificate file")
     .option(["", "key"], "Optional. Path to certificate key .pem file")
+    .option(
+      ["", "inMemoryPersistence"],
+      "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost."
+    )
     .option(
       ["d", "debug"],
       "Optional. Enable debug log by providing a valid local file path as log destination"
@@ -118,6 +122,13 @@ export default class BlobEnvironment implements IBlobEnvironment {
     return false;
   }
 
+  public inMemoryPersistence(): boolean {
+    if (this.flags.inMemoryPersistence !== undefined) {
+      return true;
+    }
+    return false;
+  }
+
   public async debug(): Promise<string | undefined> {
     if (typeof this.flags.debug === "string") {
       // Enable debug log to file

src/blob/BlobServer.ts

@@ -9,6 +9,7 @@ import IGCManager from "../common/IGCManager";
 import IRequestListenerFactory from "../common/IRequestListenerFactory";
 import logger from "../common/Logger";
 import FSExtentStore from "../common/persistence/FSExtentStore";
+import MemoryExtentStore from "../common/persistence/MemoryExtentStore";
 import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore";
 import IExtentStore from "../common/persistence/IExtentStore";
 import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore";
@@ -73,15 +74,19 @@ export default class BlobServer extends ServerBase implements ICleaner {
     // creating a new XXXDataStore class implementing IBlobMetadataStore interface
     // and replace the default LokiBlobMetadataStore
     const metadataStore: IBlobMetadataStore = new LokiBlobMetadataStore(
-      configuration.metadataDBPath
-      // logger
+      configuration.metadataDBPath,
+      configuration.isMemoryPersistence
     );
 
     const extentMetadataStore: IExtentMetadataStore = new LokiExtentMetadataStore(
-      configuration.extentDBPath
+      configuration.extentDBPath,
+      configuration.isMemoryPersistence
     );
 
-    const extentStore: IExtentStore = new FSExtentStore(
+    const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore(
+      extentMetadataStore,
+      logger
+    ) : new FSExtentStore(
       extentMetadataStore,
       configuration.persistencePathArray,
       logger

src/blob/BlobServerFactory.ts

@@ -58,7 +58,8 @@ export class BlobServerFactory {
           env.key(),
           env.pwd(),
           env.oauth(),
-          env.disableProductStyleUrl()
+          env.disableProductStyleUrl(),
+          env.inMemoryPersistence(),
         );
 
         return new SqlBlobServer(config);
@@ -79,7 +80,8 @@ export class BlobServerFactory {
           env.key(),
           env.pwd(),
           env.oauth(),
-          env.disableProductStyleUrl()
+          env.disableProductStyleUrl(),
+          env.inMemoryPersistence(),
         );
         return new BlobServer(config);
       }

src/blob/IBlobEnvironment.ts

@@ -11,4 +11,5 @@ export default interface IBlobEnvironment {
   debug(): Promise<string | boolean | undefined>;
   oauth(): string | undefined;
   disableProductStyleUrl(): boolean;
+  inMemoryPersistence(): boolean;
 }

src/blob/SqlBlobConfiguration.ts

@@ -35,7 +35,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
     key: string = "",
     pwd: string = "",
     oauth?: string,
-    disableProductStyleUrl: boolean = false
+    disableProductStyleUrl: boolean = false,
+    isMemoryPersistence: boolean = false,
   ) {
     super(
       host,
@@ -50,7 +51,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
       key,
       pwd,
       oauth,
-      disableProductStyleUrl
+      disableProductStyleUrl,
+      isMemoryPersistence,
     );
   }
 }

src/blob/SqlBlobServer.ts

@@ -8,6 +8,7 @@ import IGCManager from "../common/IGCManager";
 import IRequestListenerFactory from "../common/IRequestListenerFactory";
 import logger from "../common/Logger";
 import FSExtentStore from "../common/persistence/FSExtentStore";
+import MemoryExtentStore from "../common/persistence/MemoryExtentStore";
 import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore";
 import IExtentStore from "../common/persistence/IExtentStore";
 import SqlExtentMetadataStore from "../common/persistence/SqlExtentMetadataStore";
@@ -76,7 +77,10 @@ export default class SqlBlobServer extends ServerBase {
       configuration.sequelizeOptions
     );
 
-    const extentStore: IExtentStore = new FSExtentStore(
+    const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore(
+      extentMetadataStore,
+      logger
+    ) : new FSExtentStore(
       extentMetadataStore,
       configuration.persistenceArray,
       logger

src/blob/persistence/LokiBlobMetadataStore.ts

@@ -105,8 +105,11 @@ export default class LokiBlobMetadataStore
 
   private readonly pageBlobRangesManager = new PageBlobRangesManager();
 
-  public constructor(public readonly lokiDBPath: string) {
-    this.db = new Loki(lokiDBPath, {
+  public constructor(public readonly lokiDBPath: string, inMemory: boolean) {
+    this.db = new Loki(lokiDBPath, inMemory ? {
+      persistenceMethod: "memory"
+    } : {
+      persistenceMethod: "fs",
       autosave: true,
       autosaveInterval: 5000
     });

src/common/ConfigurationBase.ts

@@ -22,6 +22,7 @@ export default abstract class ConfigurationBase {
     public readonly pwd: string = "",
     public readonly oauth?: string,
     public readonly disableProductStyleUrl: boolean = false,
+    public readonly isMemoryPersistence: boolean = false,
   ) {}
 
   public hasCert() {

src/common/Environment.ts

@@ -70,6 +70,10 @@ args
   .option(["", "cert"], "Optional. Path to certificate file")
   .option(["", "key"], "Optional. Path to certificate key .pem file")
   .option(["", "pwd"], "Optional. Password for .pfx file")
+  .option(
+    ["", "inMemoryPersistence"],
+    "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost."
+  )
   .option(
     ["d", "debug"],
     "Optional. Enable debug log by providing a valid local file path as log destination"
@@ -155,6 +159,13 @@ export default class Environment implements IEnvironment {
     return this.flags.oauth;
   }
 
+  public inMemoryPersistence(): boolean {
+    if (this.flags.inMemoryPersistence !== undefined) {
+      return true;
+    }
+    return false;
+  }
+
   public async debug(): Promise<string | undefined> {
     if (typeof this.flags.debug === "string") {
       // Enable debug log to file

src/common/IEnvironment.ts

@@ -1,6 +1,8 @@
 import IBlobEnvironment from "../blob/IBlobEnvironment";
 import IQueueEnvironment from "../queue/IQueueEnvironment";
+import ITableEnvironment from "../table/ITableEnvironment";
 
 export default interface IEnvironment
   extends IBlobEnvironment,
-    IQueueEnvironment {}
+  IQueueEnvironment,
+  ITableEnvironment { }

src/common/VSCEnvironment.ts

@@ -109,4 +109,8 @@ export default class VSCEnvironment implements IEnvironment {
       this.workspaceConfiguration.get<boolean>("disableProductStyleUrl") || false
     );
   }
+
+  public inMemoryPersistence(): boolean {
+    return this.workspaceConfiguration.get<boolean>("inMemoryPersistence") || false;
+  }
 }

src/common/VSCServerManagerBlob.ts

@@ -87,7 +87,8 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase {
       env.key(),
       env.pwd(),
       env.oauth(),
-      env.disableProductStyleUrl()
+      env.disableProductStyleUrl(),
+      env.inMemoryPersistence(),
     );
     return config;
   }

src/common/persistence/LokiExtentMetadataStore.ts

@@ -25,8 +25,11 @@ export default class LokiExtentMetadata implements IExtentMetadataStore {
 
   private readonly EXTENTS_COLLECTION = "$EXTENTS_COLLECTION$";
 
-  public constructor(public readonly lokiDBPath: string) {
-    this.db = new Loki(lokiDBPath, {
+  public constructor(public readonly lokiDBPath: string, inMemory: boolean) {
+    this.db = new Loki(lokiDBPath, inMemory ? {
+      persistenceMethod: "memory"
+    } : {
+      persistenceMethod: "fs",
       autosave: true,
       autosaveInterval: 5000
     });