Add document size before separator

albe · Feb 21, 2021 · 7c8d829 · 7c8d829
1 parent 9bd7591
commit 7c8d829
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/src/Partition/ReadablePartition.js b/src/Partition/ReadablePartition.js
@@ -161,8 +161,8 @@ class ReadablePartition extends events.EventEmitter {
      * @returns {number} The size of the data including header, padded to 16 bytes alignment and ended with a line break.
      */
     documentWriteSize(dataSize) {
-        const padSize = (DOCUMENT_ALIGNMENT - ((dataSize + DOCUMENT_SEPARATOR.length) % DOCUMENT_ALIGNMENT)) % DOCUMENT_ALIGNMENT;
-        return dataSize + DOCUMENT_SEPARATOR.length + padSize + DOCUMENT_HEADER_SIZE;
+        const padSize = (DOCUMENT_ALIGNMENT - ((dataSize + 4 + DOCUMENT_SEPARATOR.length) % DOCUMENT_ALIGNMENT)) % DOCUMENT_ALIGNMENT;
+        return dataSize + DOCUMENT_SEPARATOR.length + 4 + padSize + DOCUMENT_HEADER_SIZE;
     }
 
     /**
@@ -328,12 +328,12 @@ class ReadablePartition extends events.EventEmitter {
 
         const separatorSize = DOCUMENT_SEPARATOR.length;
         // Optimization if we are at an exact document boundary, where we can just read the document size
-        let reader/* = this.prepareReadBufferBackwards(position);
+        let reader = this.prepareReadBufferBackwards(position);
         const block = reader.buffer.toString('ascii', reader.cursor - separatorSize, reader.cursor);
         if (block === DOCUMENT_SEPARATOR) {
             const dataSize = reader.buffer.readUInt32BE(reader.cursor - separatorSize - 4);
             return position - this.documentWriteSize(dataSize);
-        }*/
+        }
 
         do {
             reader = this.prepareReadBufferBackwards(position - separatorSize);

diff --git a/src/Partition/WritablePartition.js b/src/Partition/WritablePartition.js
@@ -8,7 +8,7 @@ const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
 const DOCUMENT_HEADER_SIZE = 16;
 const DOCUMENT_ALIGNMENT = 4;
 const DOCUMENT_SEPARATOR = "\x00\x00\x1E\n";
-const DOCUMENT_PAD = ' '.repeat(16 - DOCUMENT_SEPARATOR.length) + DOCUMENT_SEPARATOR;
+const DOCUMENT_PAD = ' '.repeat(16 - 4 - DOCUMENT_SEPARATOR.length)/* + DOCUMENT_SEPARATOR*/;
 
 const NES_EPOCH = new Date('2020-01-01T00:00:00');
 
@@ -17,8 +17,8 @@ const NES_EPOCH = new Date('2020-01-01T00:00:00');
  * @returns {string} The data padded to 16 bytes alignment and ended with \0x1E (record separator) and a line break.
  */
 function padData(dataSize) {
-    const padSize = (DOCUMENT_ALIGNMENT - ((dataSize + DOCUMENT_SEPARATOR.length) % DOCUMENT_ALIGNMENT)) % DOCUMENT_ALIGNMENT;
-    return DOCUMENT_PAD.substr(-padSize - DOCUMENT_SEPARATOR.length);
+    const padSize = (DOCUMENT_ALIGNMENT - ((dataSize + 4 + DOCUMENT_SEPARATOR.length) % DOCUMENT_ALIGNMENT)) % DOCUMENT_ALIGNMENT;
+    return DOCUMENT_PAD.substr(0, padSize);
 }
 
 /**
@@ -219,6 +219,11 @@ class WritablePartition extends ReadablePartition {
         let bytesWritten = 0;
         bytesWritten += fs.writeSync(this.fd, dataHeader);
         bytesWritten += fs.writeSync(this.fd, data);
+        bytesWritten += fs.writeSync(this.fd, padData(dataSize));
+        const dataSizeBuffer = Buffer.alloc(4);
+        dataSizeBuffer.writeUInt32BE(dataSize, 0);
+        bytesWritten += fs.writeSync(this.fd, dataSizeBuffer);
+        bytesWritten += fs.writeSync(this.fd, DOCUMENT_SEPARATOR);
         if (typeof callback === 'function') {
             process.nextTick(callback);
         }
@@ -241,6 +246,10 @@ class WritablePartition extends ReadablePartition {
         let bytesWritten = 0;
         bytesWritten += this.writeDocumentHeader(this.writeBuffer, this.writeBufferCursor, dataSize, sequenceNumber);
         bytesWritten += this.writeBuffer.write(data, this.writeBufferCursor + bytesWritten, 'utf8');
+        bytesWritten += this.writeBuffer.write(padData(dataSize), this.writeBufferCursor + bytesWritten, 'utf8');
+        this.writeBuffer.writeUInt32BE(dataSize, this.writeBufferCursor + bytesWritten);
+        bytesWritten += 4;
+        bytesWritten += this.writeBuffer.write(DOCUMENT_SEPARATOR, this.writeBufferCursor + bytesWritten, 'utf8');
         this.writeBufferCursor += bytesWritten;
         this.writeBufferDocuments++;
         if (typeof callback === 'function') {
@@ -268,7 +277,6 @@ class WritablePartition extends ReadablePartition {
         const dataSize = Buffer.byteLength(data, 'utf8');
         assert(dataSize <= 64 * 1024 * 1024, 'Document is too large! Maximum is 64 MB');
 
-        data += padData(dataSize);
         const dataPosition = this.size;
         if (dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5) {
             this.size += this.writeUnbuffered(data, dataSize, sequenceNumber, callback);