The 'IResource.read' method can now return a 'stream.Readable' in ord…

…er to optimize the read/write process
OpenMarshal · May 27, 2017 · d6b1b3e · d6b1b3e
1 parent 4f73875
commit d6b1b3e
Show file tree

Hide file tree

Showing 25 changed files with 213 additions and 73 deletions.
diff --git a/lib/manager/VirtualFSManager.js b/lib/manager/VirtualFSManager.js
@@ -15,8 +15,10 @@ var VirtualFSManager = (function () {
             properties: resource.properties
         };
         result.name = resource.name;
-        if (resource.content)
+        if (resource.content) {
             result.content = resource.content;
+            result.len = resource.len;
+        }
         return result;
     };
     VirtualFSManager.prototype.unserialize = function (data, obj) {
@@ -30,8 +32,10 @@ var VirtualFSManager = (function () {
         }
         if (obj.type.isFile) {
             var rs = new VirtualFile_1.VirtualFile(data.name, null, this);
-            if (data.content)
-                rs.content = new Buffer(data.content);
+            if (data.content) {
+                rs.content = data.content;
+                rs.len = data.len;
+            }
             rs.dateCreation = data.dateCreation;
             rs.dateLastModified = data.dateLastModified;
             rs.lockBag.locks = data.locks;

diff --git a/lib/resource/IResource.d.ts b/lib/resource/IResource.d.ts
@@ -1,3 +1,5 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
 import { FSManager } from '../manager/FSManager';
 import { XMLElement } from '../helper/XML';
 import { LockKind } from './lock/LockKind';
@@ -29,7 +31,7 @@ export interface IResource {
     isOnTheSameFSWith(resource: IResource, callback: ReturnCallback<boolean>): any;
     append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
     write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
-    read(targetSource: boolean, callback: ReturnCallback<Int8Array>): any;
+    read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): any;
     mimeType(targetSource: boolean, callback: ReturnCallback<string>): any;
     size(targetSource: boolean, callback: ReturnCallback<number>): any;
     getLocks(callback: ReturnCallback<Lock[]>): any;

diff --git a/lib/resource/physical/PhysicalFile.d.ts b/lib/resource/physical/PhysicalFile.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, ResourceType } from '../IResource';
+import { Readable } from 'stream';
 import { PhysicalResource } from './PhysicalResource';
 import { FSManager } from '../../manager/FSManager';
 export declare class PhysicalFile extends PhysicalResource {
@@ -8,7 +10,7 @@ export declare class PhysicalFile extends PhysicalResource {
     delete(callback: SimpleCallback): void;
     append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
     write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
-    read(targetSource: boolean, callback: ReturnCallback<Int8Array>): void;
+    read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): void;
     mimeType(targetSource: boolean, callback: ReturnCallback<string>): void;
     size(targetSource: boolean, callback: ReturnCallback<number>): void;
     addChild(resource: IResource, callback: SimpleCallback): void;

diff --git a/lib/resource/physical/PhysicalFolder.d.ts b/lib/resource/physical/PhysicalFolder.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, ResourceType } from '../IResource';
+import { Readable } from 'stream';
 import { ResourceChildren } from '../std/ResourceChildren';
 import { PhysicalResource } from './PhysicalResource';
 import { FSManager } from '../../manager/FSManager';
@@ -10,7 +12,7 @@ export declare class PhysicalFolder extends PhysicalResource {
     delete(callback: SimpleCallback): void;
     append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
     write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
-    read(targetSource: boolean, callback: ReturnCallback<Int8Array>): void;
+    read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): void;
     mimeType(targetSource: boolean, callback: ReturnCallback<string>): void;
     size(targetSource: boolean, callback: ReturnCallback<number>): void;
     addChild(resource: IResource, callback: SimpleCallback): void;

diff --git a/lib/resource/physical/PhysicalResource.d.ts b/lib/resource/physical/PhysicalResource.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, Return2Callback, ResourceType } from '../IResource';
+import { Readable } from 'stream';
 import { StandardResource } from '../std/StandardResource';
 import { FSManager } from '../../manager/FSManager';
 export declare abstract class PhysicalResource extends StandardResource {
@@ -13,7 +15,7 @@ export declare abstract class PhysicalResource extends StandardResource {
     abstract type(callback: ReturnCallback<ResourceType>): any;
     abstract append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
     abstract write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
-    abstract read(targetSource: boolean, callback: ReturnCallback<Int8Array>): any;
+    abstract read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): any;
     abstract mimeType(targetSource: boolean, callback: ReturnCallback<string>): any;
     abstract size(targetSource: boolean, callback: ReturnCallback<number>): any;
     abstract addChild(resource: IResource, callback: SimpleCallback): any;

diff --git a/lib/resource/std/RootResource.d.ts b/lib/resource/std/RootResource.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, Return2Callback, ResourceType } from '../IResource';
+import { Readable } from 'stream';
 import { StandardResource } from './StandardResource';
 import { ResourceChildren } from './ResourceChildren';
 export declare class RootResource extends StandardResource {
@@ -12,7 +14,7 @@ export declare class RootResource extends StandardResource {
     type(callback: ReturnCallback<ResourceType>): void;
     append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
     write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
-    read(targetSource: boolean, callback: ReturnCallback<Int8Array>): void;
+    read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): void;
     mimeType(targetSource: boolean, callback: ReturnCallback<string>): void;
     size(targetSource: boolean, callback: ReturnCallback<number>): void;
     addChild(resource: IResource, callback: SimpleCallback): void;

diff --git a/lib/resource/std/StandardResource.d.ts b/lib/resource/std/StandardResource.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, ReturnCallback, SimpleCallback, Return2Callback, ResourceType, ResourcePropertyValue } from '../IResource';
+import { Readable } from 'stream';
 import { FSManager } from '../../manager/FSManager';
 import { LockKind } from '../lock/LockKind';
 import { LockBag } from '../lock/LockBag';
@@ -31,7 +33,7 @@ export declare abstract class StandardResource implements IResource {
     abstract rename(newName: string, callback: Return2Callback<string, string>): any;
     abstract append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
     abstract write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
-    abstract read(targetSource: boolean, callback: ReturnCallback<Int8Array>): any;
+    abstract read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): any;
     abstract mimeType(targetSource: boolean, callback: ReturnCallback<string>): any;
     abstract size(targetSource: boolean, callback: ReturnCallback<number>): any;
     creationDate(callback: ReturnCallback<number>): void;

diff --git a/lib/resource/virtual/VirtualFile.d.ts b/lib/resource/virtual/VirtualFile.d.ts
@@ -1,13 +1,22 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, ResourceType } from '../IResource';
+import { Readable, ReadableOptions } from 'stream';
 import { VirtualResource } from './VirtualResource';
 import { FSManager } from '../../manager/FSManager';
+export declare class VirtualFileReadable extends Readable {
+    contents: Int8Array[];
+    blockIndex: number;
+    constructor(contents: Int8Array[], options?: ReadableOptions);
+    _read(size: number): void;
+}
 export declare class VirtualFile extends VirtualResource {
-    content: Int8Array;
+    content: Int8Array[];
+    len: number;
     constructor(name: string, parent?: IResource, fsManager?: FSManager);
     type(callback: ReturnCallback<ResourceType>): void;
     append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
     write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
-    read(targetSource: boolean, callback: ReturnCallback<Int8Array>): void;
+    read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): void;
     mimeType(targetSource: boolean, callback: ReturnCallback<string>): void;
     size(targetSource: boolean, callback: ReturnCallback<number>): void;
     addChild(resource: IResource, callback: SimpleCallback): void;

diff --git a/lib/resource/virtual/VirtualFile.js b/lib/resource/virtual/VirtualFile.js
@@ -11,43 +11,64 @@ var __extends = (this && this.__extends) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 var IResource_1 = require("../IResource");
+var stream_1 = require("stream");
 var VirtualResource_1 = require("./VirtualResource");
 var Errors_1 = require("../../Errors");
 var mimeTypes = require("mime-types");
+var VirtualFileReadable = (function (_super) {
+    __extends(VirtualFileReadable, _super);
+    function VirtualFileReadable(contents, options) {
+        var _this = _super.call(this, options) || this;
+        _this.contents = contents;
+        _this.blockIndex = -1;
+        return _this;
+    }
+    VirtualFileReadable.prototype._read = function (size) {
+        while (true) {
+            ++this.blockIndex;
+            if (this.blockIndex >= this.contents.length) {
+                this.push(null);
+                break;
+            }
+            if (!this.push(this.contents[this.blockIndex]))
+                break;
+        }
+    };
+    return VirtualFileReadable;
+}(stream_1.Readable));
+exports.VirtualFileReadable = VirtualFileReadable;
 var VirtualFile = (function (_super) {
     __extends(VirtualFile, _super);
     function VirtualFile(name, parent, fsManager) {
         var _this = _super.call(this, name, parent, fsManager) || this;
-        _this.content = new Buffer(0);
+        _this.content = [];
+        _this.len = 0;
         return _this;
     }
     VirtualFile.prototype.type = function (callback) {
         callback(null, IResource_1.ResourceType.File);
     };
     VirtualFile.prototype.append = function (data, targetSource, callback) {
-        var newContent = new Int8Array(this.content.length + data.length);
-        for (var i = 0; i < this.content.length; ++i)
-            newContent[i] = this.content[i];
-        for (var i = 0; i < data.length; ++i)
-            newContent[i + this.content.length] = data[i];
-        this.content = newContent;
+        this.content.push(data);
+        this.len += data.length;
         this.updateLastModified();
         callback(null);
     };
     VirtualFile.prototype.write = function (data, targetSource, callback) {
-        this.content = data;
+        this.content = [data];
+        this.len = data.length;
         this.updateLastModified();
         callback(null);
     };
     VirtualFile.prototype.read = function (targetSource, callback) {
-        callback(null, this.content);
+        callback(null, new VirtualFileReadable(this.content));
     };
     VirtualFile.prototype.mimeType = function (targetSource, callback) {
         var mt = mimeTypes.lookup(this.name);
         callback(null, mt ? mt : 'application/octet-stream');
     };
     VirtualFile.prototype.size = function (targetSource, callback) {
-        callback(null, this.content.length);
+        callback(null, this.len);
     };
     VirtualFile.prototype.addChild = function (resource, callback) {
         callback(Errors_1.Errors.InvalidOperation);

diff --git a/lib/resource/virtual/VirtualFolder.d.ts b/lib/resource/virtual/VirtualFolder.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, ResourceType } from '../IResource';
+import { Readable } from 'stream';
 import { ResourceChildren } from '../std/ResourceChildren';
 import { VirtualResource } from './VirtualResource';
 import { FSManager } from '../../manager/FSManager';
@@ -8,7 +10,7 @@ export declare class VirtualFolder extends VirtualResource {
     type(callback: ReturnCallback<ResourceType>): void;
     append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
     write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): void;
-    read(targetSource: boolean, callback: ReturnCallback<Int8Array>): void;
+    read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): void;
     mimeType(targetSource: boolean, callback: ReturnCallback<string>): void;
     size(targetSource: boolean, callback: ReturnCallback<number>): void;
     addChild(resource: IResource, callback: SimpleCallback): void;

diff --git a/lib/resource/virtual/VirtualResource.d.ts b/lib/resource/virtual/VirtualResource.d.ts
@@ -1,4 +1,6 @@
+/// <reference types="node" />
 import { IResource, SimpleCallback, ReturnCallback, Return2Callback, ResourceType } from '../IResource';
+import { Readable } from 'stream';
 import { FSManager } from '../../manager/FSManager';
 import { StandardResource } from '../std/StandardResource';
 export declare abstract class VirtualResource extends StandardResource {
@@ -12,7 +14,7 @@ export declare abstract class VirtualResource extends StandardResource {
     abstract type(callback: ReturnCallback<ResourceType>): any;
     abstract append(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
     abstract write(data: Int8Array, targetSource: boolean, callback: SimpleCallback): any;
-    abstract read(targetSource: boolean, callback: ReturnCallback<Int8Array>): any;
+    abstract read(targetSource: boolean, callback: ReturnCallback<Int8Array | Readable>): any;
     abstract mimeType(targetSource: boolean, callback: ReturnCallback<string>): any;
     abstract size(targetSource: boolean, callback: ReturnCallback<number>): any;
     abstract addChild(resource: IResource, callback: SimpleCallback): any;

diff --git a/lib/server/commands/Copy.js b/lib/server/commands/Copy.js
@@ -52,7 +52,21 @@ function copy(arg, source, rDest, destination, callback) {
                                     return;
                                 }
                                 source.read(true, function (e, data) { return _(e, function () {
-                                    dest.write(data, true, function (e) { return _(e, next); });
+                                    if (data.readable) {
+                                        var rdata = data;
+                                        var isFirst_1 = true;
+                                        rdata.on('data', function (chunk) {
+                                            if (isFirst_1) {
+                                                isFirst_1 = false;
+                                                dest.write(new Int8Array(0), true, function (e) { return _(e, function () { }); });
+                                            }
+                                            else
+                                                dest.append(chunk, true, function (e) { return _(e, next); });
+                                        });
+                                        rdata.on('end', next);
+                                    }
+                                    else
+                                        dest.write(data, true, function (e) { return _(e, next); });
                                 }); });
                                 function next() {
                                     if (!type.isDirectory) {

diff --git a/lib/server/commands/Get.js b/lib/server/commands/Get.js
@@ -16,16 +16,25 @@ function default_1(arg, callback) {
                         arg.setCode(WebDAVRequest_1.HTTPCodes.MethodNotAllowed);
                     else {
                         arg.setCode(WebDAVRequest_1.HTTPCodes.OK);
-                        var content = c;
-                        if (c === undefined || c === null)
-                            content = new Buffer(0);
-                        else if (c.constructor === Boolean || c.constructor === Number)
-                            content = c.toString();
-                        else if (c.constructor === Int8Array)
-                            content = new Buffer(c);
-                        else
-                            content = c;
-                        arg.response.write(content);
+                        if (c.readable) {
+                            var rdata = c;
+                            var isFirst = true;
+                            rdata.on('end', callback);
+                            rdata.pipe(arg.response);
+                            return;
+                        }
+                        else {
+                            var content = c;
+                            if (c === undefined || c === null)
+                                content = new Buffer(0);
+                            else if (c.constructor === Boolean || c.constructor === Number)
+                                content = c.toString();
+                            else if (c.constructor === Int8Array)
+                                content = new Buffer(c);
+                            else
+                                content = c;
+                            arg.response.write(content);
+                        }
                     }
                     callback();
                 }); });

diff --git a/src/manager/VirtualFSManager.ts b/src/manager/VirtualFSManager.ts
@@ -12,19 +12,19 @@ export class VirtualFSManager implements FSManager
 
     serialize(resource : any, obj : SerializedObject) : object
     {
-        const result : {
-            name,
-            content
-        } = {
+        const result : any = {
             dateCreation: resource.dateCreation,
             dateLastModified: resource.dateLastModified,
             locks: resource.lockBag.locks,
             properties: resource.properties
-        } as any;
+        };
 
         result.name = resource.name;
         if(resource.content)
+        {
             result.content = resource.content;
+            result.len = resource.len;
+        }
 
         return result;
     }
@@ -44,7 +44,10 @@ export class VirtualFSManager implements FSManager
         {
             const rs = new VirtualFile(data.name, null, this);
             if(data.content)
-                rs.content = new Buffer(data.content);
+            {
+                rs.content = data.content;
+                rs.len = data.len;
+            }
             rs.dateCreation = data.dateCreation;
             rs.dateLastModified = data.dateLastModified;
             rs.lockBag.locks = data.locks;

diff --git a/src/resource/IResource.ts b/src/resource/IResource.ts
@@ -1,3 +1,4 @@
+import { Readable, ReadableOptions } from 'stream'
 import { FSManager, FSPath } from '../manager/FSManager'
 import { XMLElement } from '../helper/XML'
 import { LockKind } from './lock/LockKind'
@@ -49,7 +50,7 @@ export interface IResource
     // ****************************** Content ****************************** //
     append(data : Int8Array, targetSource : boolean, callback : SimpleCallback)
     write(data : Int8Array, targetSource : boolean, callback : SimpleCallback)
-    read(targetSource : boolean, callback : ReturnCallback<Int8Array>)
+    read(targetSource : boolean, callback : ReturnCallback<Int8Array|Readable>)
     mimeType(targetSource : boolean, callback : ReturnCallback<string>)
     size(targetSource : boolean, callback : ReturnCallback<number>)
 

diff --git a/src/resource/physical/PhysicalFile.ts b/src/resource/physical/PhysicalFile.ts
@@ -1,4 +1,5 @@
 import { IResource, SimpleCallback, ReturnCallback, ResourceType } from '../IResource'
+import { Readable, ReadableOptions } from 'stream'
 import { PhysicalResource } from './PhysicalResource'
 import { FSManager } from '../../manager/FSManager'
 import { Errors } from '../../Errors'
@@ -72,7 +73,7 @@ export class PhysicalFile extends PhysicalResource
             }
         });
     }
-    read(targetSource : boolean, callback : ReturnCallback<Int8Array>)
+    read(targetSource : boolean, callback : ReturnCallback<Int8Array|Readable>)
     {
         fs.readFile(this.realPath, callback);
     }