feat(core): added mkdirp-on-write support to memorydisk and localdisk

README.md

@@ -184,16 +184,19 @@ for inline documentation and types.
     -   `Readable` stream passed to `MemoryDisk`/`LocalDisk`
     -   Properly handled symlinks in directory listings for `MemoryDisk`/`LocalDisk`
     -   Proper errors from bad permissions for `MemoryDisk`/`LocalDisk`
+    -   Multiple writes to the same file do truncate
 -   [ ] Document the `Disk` API.
 -   [ ] Document the `DiskManager` API.
 -   [ ] Support `rimraf` for directories.
--   [ ] Fix `FSDisk` (backend to `MemoryDisk` and `LocalDisk`) to `mkdirp` path to file to mirror s3 behaviour.
--   [ ] When a file is deleted on `FSDisk` and its the only file in the directory, delete the directory, following the
-        path backwards to do the same to get rid of all fs tree leaves.
 -   [ ] Support `force` for delete which doesn't to mimic `rm -f` which doesn't fail if the file isn't found.
 -   [ ] Separate driver from remaining options so that the `driver` options doesn't have to be passed to `*Disk`
         constructors.
 -   [ ] Wrap all unknown errors in an `UnknownDiskError` (maybe using `VError`?)
+-   [ ] Ensure that when memfs is used, we always use the posix path module even on a win32 host FS (or otherwise
+        verify that on win32, memfs uses win32 paths).
+-   [ ] Proxy read and write streams so that errors emitted on streams can be wrapped
+-   [ ] Upgrade `CodedError` to also contain a reference to the original system/driver error
+-   [ ] LocalDisk config setting to filter directory listings by only accessible files/directories.
 
 ## Development
 

src/drivers/memory/VolumeFSModule.ts

@@ -93,7 +93,7 @@ export class VolumeFSModule implements AsyncFSModule {
                     if (error) {
                         reject(error);
                     } else if (data && Array.isArray(data)) {
-                        // Use `as` here to force
+                        // transformVolumeDataToString can handle any TDataOut or Dirent even when mixed
                         const results: string[] = (data as (
                             | TDataOut
                             | Dirent)[]).map(transformVolumeDataToString);
@@ -141,4 +141,22 @@ export class VolumeFSModule implements AsyncFSModule {
             });
         });
     }
+
+    public access(path: string, mode?: number): Promise<void> {
+        return new Promise((resolve, reject) => {
+            const fulfill = (error?: IError): void => {
+                if (error) {
+                    reject(error);
+                } else {
+                    resolve();
+                }
+            };
+
+            if (typeof mode === 'number') {
+                this.volume.access(path, mode, fulfill);
+            } else {
+                this.volume.access(path, fulfill);
+            }
+        });
+    }
 }

src/lib/fs/FSDisk.ts

@@ -10,7 +10,7 @@ import {
 } from '../../errors';
 import { DiskConfig, DiskListingObject, DiskObjectType } from '../types';
 import { AsyncFSModule } from './types';
-import { pipeStreams, streamToBuffer } from '../utils';
+import { pipeStreams } from '../utils';
 
 /**
  * Represents a disk that uses a traditional filesystem. Expects an `AsyncFSModule` which is essentially just
@@ -66,6 +66,22 @@ export abstract class FSDisk extends Disk {
         return rootPath;
     }
 
+    /**
+     * @inheritDoc
+     */
+    public async read(pathOnDisk: string): Promise<Buffer> {
+        try {
+            return await this.fs.readFile(this.getFullPath(pathOnDisk));
+        } catch (error) {
+            if (error.code === 'EISDIR') {
+                throw new NotAFileError(pathOnDisk);
+            } else if (error.code === 'ENOENT') {
+                throw new NotFoundError(pathOnDisk);
+            }
+            throw error;
+        }
+    }
+
     /**
      * @inheritDoc
      */
@@ -90,45 +106,35 @@ export abstract class FSDisk extends Disk {
     }
 
     /**
-     * @inheritDoc
+     * Perform a write operation which involves some prep (creating the leading directory) and the transformation of
+     * certain "expected" errors into more understandable errors for the consumer of the library.
+     *
+     * @param pathOnDisk
+     * @param execute
      */
-    public async createWriteStream(pathOnDisk: string): Promise<Writable> {
+    private async prepareAndExecuteWrite<T>(
+        pathOnDisk: string,
+        execute: (fullPath: string) => Promise<T>,
+    ): Promise<T> {
         const fullPath = this.getFullPath(pathOnDisk);
-        let stats = null;
         try {
-            stats = await this.fs.stat(fullPath);
+            // Ensure the path leading up to the file exists as a directory.
+            await this.fs.mkdirp(path.dirname(fullPath));
         } catch (error) {
-            if (error.code !== 'ENOENT') {
-                // If the file doesn't exists, that's good! Continue. Otherwise throw.
-                throw error;
-            }
-        }
-        // If the file exists and it's not a file, fail.
-        if (stats && !stats.isFile()) {
-            throw new NotWritableDestinationError(pathOnDisk);
-        }
-
-        try {
-            return this.fs.createWriteStream(fullPath);
-        } catch (error) {
-            if (error.code === 'EISDIR') {
+            // EEXIST is what node `fs` and `fs-extra` will throw if the leading path exists as a non-directory.
+            // ENOTDIR is what `memfs` throws if the leading path exists as a non-directory.
+            // Both of these will catch trying to write to `/foo/bar.txt` where `/foo` is a file.
+            if (error.code === 'EEXIST' || error.code === 'ENOTDIR') {
                 throw new NotWritableDestinationError(pathOnDisk);
             }
             throw error;
         }
-    }
-
-    /**
-     * @inheritDoc
-     */
-    public async read(pathOnDisk: string): Promise<Buffer> {
         try {
-            return await this.fs.readFile(this.getFullPath(pathOnDisk));
+            // Execute the write, capturing and wrapping important errors as necessary.
+            return await execute(fullPath);
         } catch (error) {
-            if (error.code === 'EISDIR') {
-                throw new NotAFileError(pathOnDisk);
-            } else if (error.code === 'ENOENT') {
-                throw new NotFoundError(pathOnDisk);
+            if (error.code === 'EISDIR' || error.code === 'EACCES') {
+                throw new NotWritableDestinationError(pathOnDisk);
             }
             throw error;
         }
@@ -141,20 +147,35 @@ export abstract class FSDisk extends Disk {
         pathOnDisk: string,
         body: Buffer | string | Readable,
     ): Promise<void> {
-        try {
-            if (typeof body === 'object' && body instanceof Readable) {
-                // If we were provided with a stream, we'll delegate to `createWriteStream`
-                const writeStream = await this.createWriteStream(pathOnDisk);
-                await pipeStreams(body, writeStream);
-            } else {
-                await this.fs.writeFile(this.getFullPath(pathOnDisk), body);
-            }
-        } catch (error) {
-            if (error.code === 'EISDIR') {
-                throw new NotWritableDestinationError(pathOnDisk);
-            }
-            throw error;
-        }
+        return this.prepareAndExecuteWrite(
+            pathOnDisk,
+            async (fullPath: string): Promise<void> => {
+                if (typeof body === 'object' && body instanceof Readable) {
+                    // If we were provided with a stream, we'll open up a stream and pipe to it.
+                    const writeStream = await this.fs.createWriteStream(
+                        fullPath,
+                    );
+                    await pipeStreams(body, writeStream);
+                } else {
+                    // Otherwise we have a string or a buffer so we can just write the contents using writeFile
+                    await this.fs.writeFile(fullPath, body);
+                }
+            },
+        );
+    }
+
+    /**
+     * Create a write stream to a file on the disk. Note that errors like EISDIR aren't thrown by the createWriteStream
+     * function and instead are emitted as errors on the stream so those will be raw unwrapped errors.
+     * @inheritDoc
+     */
+    public async createWriteStream(pathOnDisk: string): Promise<Writable> {
+        return this.prepareAndExecuteWrite(
+            pathOnDisk,
+            async (fullPath: string): Promise<Writable> => {
+                return this.fs.createWriteStream(fullPath);
+            },
+        );
     }
 
     /**

src/lib/fs/types.ts

@@ -10,4 +10,5 @@ export interface AsyncFSModule {
     createWriteStream: (path: string) => stream.Writable;
     unlink: (file: string) => Promise<void>;
     mkdirp: (dirPath: string) => Promise<void>;
+    access: (file: string, mode?: number) => Promise<void>;
 }