diff --git a/README.md b/README.md index 753837c..1ae68c8 100755 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ A few notes on the formats: 2. [mz3](https://github.com/neurolabusc/surf-ice/tree/master/mz3) tested as both GZip compressed and raw formats. 3. [OBJ](https://brainder.org/tag/wavefront-obj/) is an ASCII format, so a choice must be made regarding file size and precision. 4. [STL](http://paulbourke.net/dataformats/stl/) format does not re-use vertices. The resulting mesh will appear faceted and use more GPU resources unless one applies a computationally expensive operation to weld vertices. + 5. [jmsh](https://en.wikipedia.org/wiki/JMesh) is a JSON-based general purpose geometry/mesh-data container based on the [JMesh](https://github.com/NeuroJSON/jmesh/blob/master/JMesh_specification.md) and [JData](https://github.com/NeuroJSON/jdata/blob/master/JData_specification.md) specifications; it is human-readable and widely parsable + 6. [bmsh](https://en.wikipedia.org/wiki/JMesh) is a binary-JSON ([BJData - Draft 2](https://github.com/NeuroJSON/bjdata/blob/Draft_2/Binary_JData_Specification.md)) based mesh-data container based on the JMesh specification; both `.bmsh` and `.jmsh` support data-level compression + 7. [json](http://json.org) is a minimized plain JSON file using [JMesh](https://github.com/NeuroJSON/jmesh/blob/master/JMesh_specification.md) annotations without compression ![formats](formats.png) @@ -15,14 +18,19 @@ A few notes on the formats: This is a simple node.js function that you can replicate on your own computer: ``` -$ npm install gifti-reader-js +$ npm install gifti-reader-js atob numjs pako buffer $ git clone https://github.com/neurolabusc/MeshFormatsJS.git $ cd MeshFormatsJS $ node ./meshtest.js -gifti.gii Size 4384750 Time 683 -gz.mz3 Size 3259141 Time 424 -raw.mz3 Size 5898280 Time 49 -obj.obj Size 13307997 Time 498 -stl.stl Size 16384084 Time 249 +gifti.gii Size 4384750 Time 2111 +gz.mz3 Size 3259141 Time 541 +raw.mz3 Size 5898280 Time 31 +obj.obj Size 13307997 Time 5318 +stl.stl Size 16384084 Time 1075 +zlib.jmsh Size 4405604 Time 660 +zlib.bmsh Size 3259049 Time 479 +raw.min.json Size 12325881 Time 1239 +raw.bmsh Size 5898902 Time 38 + ``` diff --git a/lib/bjdata.js b/lib/bjdata.js new file mode 100644 index 0000000..0e202ee --- /dev/null +++ b/lib/bjdata.js @@ -0,0 +1,655 @@ +// Licensed to Pioneers in Engineering under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Pioneers in Engineering licenses +// this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License + +// This file was (hastily) ported from lua-ubjson. +// The code structure there will make probably make more sense. +// Multiple return values there have been manually transformed into arrays +// here, and generally make the code harder to understand. +// There are also a few lingering Lua-isms, like keeping track of stack depth +// for error handling, excessive use of nil / null, variable names suggesting +// string and buffers are the same thing, and ambiguity about what's an array +// and what's an object. +// The comments also have not been updated. Comments that look like they've +// been mangled by a regex probably have. + +// Global dependencies here. + +var buffer_module = require('buffer'); + +var abs = Math.abs; +var floor = Math.floor; +var dumpint = function (val, size, endian) { + var b = buffer_module.Buffer(size); + + if (val > pow2(8 * size)) { + // twos-complement encode too-large values. + val = val - pow2(8 * size); + } + if (size === 1) + endian = ''; + b['writeInt' + (8 * size) + endian].apply(b, [val, 0]); + return b; +}; + +var dumpfloat = function (val, type, endian) { + var b; + + if (type === 'd') { + b = buffer_module.Buffer(4); + b['writeFloat' + endian].apply(b, [val, 0]); + } else if (type === 'D') { + b = buffer_module.Buffer(8); + b['writeDouble' + endian].apply(b, [val, 0]); + } else { + throw 'Unexpected float type ' + type + '.'; + } + return b; +}; + +var undumpint = function(buf, offset, size, endian) { + if (size === 1) + endian = ''; + return buf['readInt' + (8 * size) + endian].apply(buf, [offset]); +}; + +var undumpfloat = function(buf, offset, type, endian) { + if (type === 'd') { + return buf['readFloat' + endian].apply(buf, [offset]); + } else if (type === 'D') { + return buf['readDouble' + endian].apply(buf, [offset]); + } else { + throw 'Unexpected float type ' + type + '.'; + } +}; + +var toBuffer = function (str) { + return buffer_module.Buffer(str); +}; + +var type = function (val) { + return typeof val; +}; +var error = function (msg) { + throw "js-bjdata: " + msg; +}; + +var insert = function(array, val) { + array.push(buffer_module.Buffer(val)); +}; + +var bufStr = function(buf, start, end) { + return buf.slice(start, end + 1).toString(); +}; + +// Calculate 2 ^ v +var pow2 = function(v) { + var out = 1; + for (var i = 0; i < v; i++) { + // Use floats, since this may end up very large. + out = out * 2.0; + } + return out; +}; + +// Mapping from maximum value -> ubjson tag +var int_maxes = [ + pow2( 8), + pow2(16), + pow2(32), + pow2(64), +]; + +var int_tags = [ + 'U', + 'u', + 'm', + 'M', +]; + +// ubjson tag -> size in bytes +var int_tag_size = { + U : 1, + i : 1, + u : 2, + I : 2, + m : 4, + l : 4, + M : 8, + L : 8, +}; + +// bjdata tag -> jdata tags +var jd_type = { + U : "uint8", + i : "int8", + u : "uint16", + I : "int16", + m : "uint32", + l : "int32", + M : "uint64", + L : "int64", + d : "float32", + D : "float64", +}; + +// bjdata tag -> jdata tags +var jd_len = { + U : 1, + i : 1, + u : 2, + I : 2, + m : 4, + l : 4, + M : 8, + L : 8, + d : 4, + D : 8, +}; + +var typedfun={ + "Float32Array":null,"Float64Array":null, + "Int8Array":null, "Uint8Array":null, + "Int16Array":null, "Uint16Array":null, + "Int32Array":null, "Uint32Array":null, + "BigInt64Array":null, "BigUint64Array":null +}; + +// Use doubles to serialize Lua numbers. +var use_double = true; + +// Get the smallest tag and size to hold a value. +function int_tag(val) { + if ( val >= 0 && val < 256 ) { + return ['U', 1]; + } + var last_key = 'i'; + var size = 1; + // Calculate the absolute value. + if ( val < 0 ) { + val = -val; + // Because twos-complement can hold slightly larger negatives than + // positives. + if ( val !== 0 ) { + val = val - 1; + } + } + for (var idx in int_maxes) { + var max = int_maxes[idx]; + if ( val > max ) { + return [last_key, size]; + } else { + last_key = int_tags[idx]; + size = size * 2; + } + } + return [last_key, size / 2]; +} + +// If val can be represented by a fixed size value type, return the tag for +// that type, otherwise return the Lua type string. +function val_tag(val) { + var t = type(val); + if ( t === 'number' ) { + t = int_tag(val)[0]; + } else if ( t === 'boolean' ) { + if ( t ) { + return 'T'; + } else { + return 'F'; + } + } else if ( t === 'null' ) { + return 'Z'; + } + return t; +} + +// Pre-declare encode_inner +var encode_inner; + +// Determines whether an table should become an array or a table. +// Also determines length, and whether the optimized container optimization can +// be applied. +// returns [use_obj, length, max_index, shared_tag, write_val] +// where +// use_obj is true iff the table should become a ubjson object (not an array). +// length is the number of entries in the array or table. +// max_index is the largest integer index. +// shared_tag is a ubjson type tag if ( the optimized container format can be +// applied, otherwise is the string 'mixed' +// write_val is a function which writes a value for the object or array. +// write_val has the same type as encode_inner +// (value, buffer, memo, depth) -> () +// where value is the value to be serialized +// buffer is a table of strings which will be concatenated together to +// produce the output +// memo is a table mapping currently entered tables to true +// depth is the recursion depth from the user's call +function array_helper(val) { + // TODO(kzentner): Handle integer tags more intelligently. + // Currently, this function only handles integers well when every integer is + // expected to have the same tag. In theory, this could produce an array for + // any integer type tag, but in practice probably will almost always only + // produce 'U'. + // Basically, this function expects val_tag to return the same tag for every + // value if the fixed-type container optimization can be applied. This is + // definitely not true. For example, [0, -1, 2] produces the tags ['U', 'i', + // 'U'], but all the entries can be represented by one signed byte. + // + var t = null; + var length = 0; + var max = 0; + + for (var k in val) { + var v = val[k]; + if ( k > max ) { + max = k; + } + if ( t === null ) { + t = val_tag(v); + } + if ( t !== val_tag(v) ) { + t = 'mixed'; + } + length = length + 1; + } + + var write_val = encode_inner; + + if ( t !== null && t.length === 1 ) { + var size = int_tag_size[t]; + if ( size ) { + write_val = function(val, buffer, memo) { + insert(buffer, dumpint(val, size, 'LE')); + }; + } else if ( t === 'd' ) { + write_val = function(val, buffer, memo) { + insert(buffer, dumpfloat(val, 'd', 'LE')); + }; + } else if ( t === 'D' ) { + write_val = function(val, buffer, memo) { + insert(buffer, dumpfloat(val, 'D', 'LE')); + }; + } else { + // Tag should be 'T', 'F', 'Z' + write_val = function(val, buffer, memo) { + }; + } + } + + // TODO(kzentner): Handle array-likes like Uint8Array's, etc. better. + // Note that isArray(new Uint8Array) == false + return [!Array.isArray(val), length, max, t, write_val]; +} + +function encode_int(val, buffer) { + var ts = int_tag(val); + var tag = ts[0]; + var size = ts[1]; + insert(buffer, tag); + insert(buffer, dumpint(val, size, 'LE')); + + // TODO(kzentner): Huge int support? +} + +function encode_inner(val, buffer, memo, depth) { + var k; + // if val in memo. Some things, Javascript makes really weird. + if ( ~memo.indexOf(val) ) { + error('Cannot serialize circular data structure.', depth); + } + if ( depth === undefined ) { + error('Depth missing.'); + } + + var t = type(val); + if ( t === 'number' ) { + if ( floor(val) === val ) { + encode_int(val, buffer); + } else { + if ( use_double ) { + insert(buffer, 'D'); + insert(buffer, dumpfloat(val, 'D', 'LE')); + } else { + insert(buffer, 'd'); + insert(buffer, dumpfloat(val, 'd', 'LE')); + } + } + } else if ( t === 'null' || t === 'undefined' ) { + insert(buffer, 'Z'); + } else if ( t === 'boolean' ) { + if ( val ) { + insert(buffer, 'T'); + } else { + insert(buffer, 'F'); + } + } else if ( t === 'string' ) { + insert(buffer, 'S'); + encode_int(val.length, buffer); + insert(buffer, val); + } else if ( t === 'object' ) { + memo.push(val); + var ulmtw = array_helper(val); + var use_obj = ulmtw[0]; + var length = ulmtw[1]; + var max = ulmtw[2]; + var tag = ulmtw[3]; + var write_val = ulmtw[4]; + if ( use_obj ) { + insert(buffer, '{'); + } else { + insert(buffer, '['); + } + + if ( tag !== null && tag.length === 1 ) { + insert(buffer, '$'); + insert(buffer, tag); + } + + insert(buffer, '#'); + encode_int(length, buffer); + + if ( use_obj ) { + for (k in val) { + var v = val[k]; + var str = k + ''; + encode_int(str.length, buffer); + insert(buffer, str); + write_val(v, buffer, memo, depth + 1); + } + } else { + for (k = 0; k <= max; k++ ) { + write_val(val[k], buffer, memo, depth + 1); + } + } + // Remove val from memo. + memo.splice(memo.indexOf(val), 1); + } +} + +function encode(value, state) { + var buffer = []; + var memo = []; + var k; + encode_inner(value, buffer, memo, 3); + var total_length = 0; + for (k in buffer) { + total_length += buffer[k].length; + } + var out = buffer_module.Buffer(total_length); + var current_offset = 0; + for (k in buffer) { + var b = buffer[k]; + b.copy(out, current_offset, 0, b.length); + current_offset += b.length; + } + return out; +} + +function decode_int(str, offset, depth, error_context) { + var c = bufStr(str, offset, offset); + var int_size = int_tag_size[c]; + if ( int_size === undefined ) { + error(error_context + ' length did not have an integer tag.', depth); + } + var i = undumpint(str, offset + 1, int_size, 'LE'); + if ( c === 'U' && i < 0 ) { + // Undo twos-complement + i = 256 + i; + } + + return [i, offset + 1 + int_size]; +} + +// Returns function with signature +// (str, offset, depth) -> val, new_offset, skip +// where str is the input string +// offset is the index into str to start reading at +// depth is the recursion depth from the user's call +// val is the read value +// new_offset is the offset after the read element +// skip is whether the object should be recognized +// (used to implement noop) +function get_read_func(tag) { + var int_size = int_tag_size[tag]; + if ( tag === 'C' ) { + int_size = 1; + } + if ( int_size !== undefined ) { + return function(str, offset, depth) { + return [undumpint(str, offset, int_size, 'LE'), offset + int_size]; + }; + } else if ( tag === 'd' ) { + return function(str, offset, depth) { + return [undumpfloat(str, offset, 'd', 'LE'), offset + 4]; + }; + } else if ( tag === 'D' ) { + return function(str, offset, depth) { + return [undumpfloat(str, offset, 'D', 'LE'), offset + 8]; + }; + } else if ( tag === 'T' ) { + return function(str, offset, depth) { + return [true, offset]; + }; + } else if ( tag === 'F' ) { + return function(str, offset, depth) { + return [false, offset]; + }; + } else if ( tag === 'Z' ) { + return function(str, offset, depth) { + return [null, offset]; + }; + } else if ( tag === 'N' ) { + return function(str, offset, depth) { + return [null, offset, true]; + }; + } else { + return null; + } +} + +// Decodes a string. Does ! read the type tag, so that it can be used to +// decode ubjson object keys. +function decode_str(str, offset, depth) { + var ls = decode_int(str, offset, depth + 1, 'String at offset ' + offset); + var str_length = ls[0]; + var str_start = ls[1]; + // Since bufStr is inclusive at of the end, -1 is needed. + return [bufStr(str, str_start, str_start + str_length - 1), str_start + str_length]; +} + +// Recursive function used to decode object. +// (str, offset, depth) -> (val, new_offset, skip) +// where str is the input string +// offset is the index into str to start reading at +// depth is the recursion depth from the user's call +// val is the read value +// new_offset is the offset after the read element +// skip is whether the object should be recognized +// (used to implement noop) +function decode_inner(str, offset, depth) { + if ( depth === null ) { + error('Depth missing'); + } + var c = bufStr(str, offset, offset); + var int_size = int_tag_size[c]; + if ( int_size !== undefined ) { + return [undumpint(str, offset + 1, int_size, 'LE'), offset + 1 + int_size]; + } else if ( c === 'C' ) { + return [undumpint(str, offset + 1, 1, 'LE'), offset + 2]; + } else if ( c === 'S' || c === 'H' ) { + // TODO(kzentner): How to handle huge numbers? + return decode_str(str, offset + 1, depth + 1) + } else if ( c === 'T' ) { + return [true, offset + 1]; + } else if ( c === 'F' ) { + return [false, offset + 1]; + } else if ( c === 'Z' ) { + return [null, offset + 1]; + } else if ( c === 'N' ) { + return [null, offset + 1, true]; + } else if ( c === 'd' ) { + return [undumpfloat(str, offset + 1, 'd', 'LE'), offset + 5]; + } else if ( c === 'D' ) { + return [undumpfloat(str, offset + 1, 'D', 'LE'), offset + 9]; + } else if ( c === '[' || c === '{' ) { + var start_offset = offset + 1; + var tag = bufStr(str, start_offset, start_offset); + var length = null; + var out; + var read_val = decode_inner; + var t = ' ' + if ( tag === '$' ) { + start_offset = start_offset + 1; + t = bufStr(str, start_offset, start_offset); + start_offset = start_offset + 1; + tag = bufStr(str, start_offset, start_offset); + read_val = get_read_func(t); + if ( read_val === null ) { + if ( c === '[' ) { + error('Type tag for non value type in array at offset ' + offset, + depth); + } else { + error('Type tag for non value type in object at offset ' + offset, + depth); + } + } + } + // TODO(kzentner): Do not construct the error message every time. + if ( tag === '#' ) { + var msg; + if ( c === '[' ) { + msg = 'Array'; + } else { + msg = 'Object'; + } + msg = msg + ' length at offset ' + offset; + var ls; + if(bufStr(str, start_offset+1, start_offset+1) == '['){ + ls=decode_inner(str, start_offset+1, depth); + length=ls[0].reduce((a, b)=> a*b, 1); + }else{ + ls = decode_int(str, start_offset + 1, depth + 1, msg); + length = ls[0]; + } + start_offset = ls[1]; + } else { + start_offset = start_offset - 1; + } + + var elt_offset = start_offset+1; + var key, val, skip; + var ke; + var ves; + var i; + if ( c === '[' ) { + out = []; + if ( length !== null ) { + elt_offset = start_offset; + let tagid=jd_type[t]; + if(tagid !== undefined){ + let type=jd_type[t]; + let bytelen=jd_len[t] * length; + let typename=type.charAt(0).toUpperCase() + type.substring(1) + "Array"; + if(type=='int64' || type=='uint64') + typename='Big'+typename; + out=new Uint8Array(Buffer.from(str.buffer, elt_offset, bytelen)); + if(typedfun[typename] == null) + typedfun[typename]=new Function('d', 'return new '+typename+'(d)'); + let typecast=typedfun[typename]; + out=typecast(out.buffer); + elt_offset+=bytelen; + }else{ + for (i = 0; i < length; i++) { + ves = read_val(str, elt_offset, depth + 1); + val = ves[0]; + elt_offset = ves[1]; + skip = ves[2]; + if ( ! skip ) { + out.push(val); + } + } + } + } else { + while ( bufStr(str, elt_offset, elt_offset) !== ']' ) { + ves = read_val(str, elt_offset, depth + 1); + val = ves[0]; + elt_offset = ves[1]; + skip = ves[2]; + if ( ! skip ) { + out.push(val); + } + } + elt_offset++; + } + } else { + out = {}; + if ( length !== null ) { + for (i = 0; i < length; i++) { + ke = decode_str(str, elt_offset, depth + 1); + key = ke[0]; + elt_offset = ke[1]; + ves = read_val(str, elt_offset, depth + 1); + val = ves[0]; + elt_offset = ves[1]; + skip = ves[2]; + if ( ! skip ) { + out[key] = val; + } + } + } else { + while ( bufStr(str, elt_offset, elt_offset) !== '}' ) { + ke = decode_str(str, elt_offset, depth + 1); + key = ke[0]; + elt_offset = ke[1]; + ves = read_val(str, elt_offset, depth + 1); + val = ves[0]; + elt_offset = ves[1]; + skip = ves[2]; + if ( ! skip ) { + out[key] = val; + } + } + elt_offset++; + } + } + return [out, elt_offset]; + } else { + error('Unrecognized type tag ' + c + ' at offset ' + offset + '.', depth); + } +} + +// Get decoded value and the offset after it in the buffer. +function decode_offset(str, offset) { + if (offset === undefined) { + offset = 0; + } + return decode_inner(str, offset, 1); +} + +// Just get the decoded value. +function decode(str, offset) { + return decode_offset(str, offset); +} + +var bjdata = { + version : 'js-bjdata 0.2', + encode : encode, + decode : decode, + decode_offset: decode_offset +}; + +module.exports = bjdata; diff --git a/lib/jdata.js b/lib/jdata.js new file mode 100644 index 0000000..54b9429 --- /dev/null +++ b/lib/jdata.js @@ -0,0 +1,227 @@ +/******************************************************************************** + JData Encoder and Decoder for JavaScript + (for JData Specification Draft 2) + + Author: Qianqian Fang + URL: http://github.com/fangq/jsdata + Live Demo: https://jsfiddle.net/fangq/7vLxjwa6/ +********************************************************************************/ + +class jdata{ + + constructor(data = {}, options = {}) { + this.opt = options; + this.data = data; + this.const={_Inf_:Infinity,_NaN_:NaN}; + this.typedfun={ + "Float32Array":null,"Float64Array":null, + "Int8Array":null, "Uint8Array":null, + "Int16Array":null, "Uint16Array":null, + "Int32Array":null, "Uint32Array":null, + "BigInt64Array":null, "BigUint64Array":null + }; + this._zipper = (typeof pako !== 'undefined' + ? pako + : require('pako')); + this._nj = (typeof nj !== 'undefined' + ? nj + : require('numjs')); + this._nj.NdArray.prototype.toJSON = function(){ + return JSON.stringify(this.tolist(),function(k,v){ + if (typeof v === 'bigint') + return '~~'+v.toString()+'~~'; + return v; + }); + }; + } + + encode(){ + this.data=this._encode(this.data); + return this; + } + + decode(){ + this.data=this._decode(this.data); + return this; + } + + tojson(){ + return JSON.stringify(this.data, this._exportfilter.bind(this),'\t').replace(/\\/g, '') + .replace(/\"\[/g, '[') + .replace(/\]\"/g,']') + .replace(/\"\{/g, '{') + .replace(/\}\"/g,'}') + .replace(/\"~~/g, '') + .replace(/~~\"/g, ''); + } + + zip(buf, method){ + if(method!=='zlib' || method!=='gzip') + method='zlib'; + if(method==='zlib') + return btoa(this._zipper.deflate(new Uint8Array(buf), { to: 'string' })); + else if(method==='gzip') + return btoa(this._zipper.gzip(new Uint8Array(buf), { to: 'string' })); + } + + unzip(str, method){ + if(method==='zlib') + return this._zipper.inflate(str); + else if(method==='gzip') + return this._zipper.ungzip(str); + else + throw "compression method not supported"; + } + + _istypedarray(obj){ + return !!obj && obj.byteLength !== undefined; + } + + static _str2hex(str){ + str = encodeURIComponent(str).split('%').join(''); + return str.toLowerCase(); + } + + _exportfilter(k,v){ + if (typeof v === 'bigint'){ + return v.toString(); + }else if(v instanceof Array){ + return JSON.stringify(v); + }else if (this._istypedarray(v)){ + return Array.apply([], v); + }else if(v instanceof this._nj.NdArray){ + return v.tolist(); + }else if(v instanceof Map) + return Array.from(v.entries()); + return v; + } + + _encode(obj){ + let newobj=obj; + if(typeof obj == 'number'){ + if(obj === Infinity){ + return "_Inf_"; + }else if (obj === -Infinity){ + return "-_Inf_"; + }else if (obj !== obj){ + return "_NaN_"; + } + }else if(obj instanceof Array){ + obj.forEach(function(e,idx,orig){ + orig[idx]=this._encode(e); + }.bind(this)); + newobj=obj; + }else if(this._istypedarray(obj)){ + let dtype=Object.prototype.toString.call(obj); + if(dtype=='[object ArrayBuffer]'){ + obj=new Uint8Array(obj); + dtype=Object.prototype.toString.call(obj); + } + dtype=dtype.replace(/\[object /,'').replace(/^Big/,'') + .replace(/Array\]/,'').replace(/Clamped/,''); + dtype=dtype.replace(/Float32/,'single').replace(/Float64/,'double'); + newobj={_ArrayType_: dtype.toLowerCase() ,_ArraySize_:obj.length}; + if((dtype == 'Int64' || dtype=='Uint64') || this.opt!==undefined && this.opt.hasOwnProperty('compression')){ + if(this.opt.compression === undefined) + newobj._ArrayZipType_='zlib'; + else + newobj._ArrayZipType_=this.opt.compression; + newobj._ArrayZipSize_=obj.length; + newobj._ArrayZipData_=this.zip(obj.buffer,newobj._ArrayZipType_); + }else{ + newobj._ArrayData_=Array.from(obj); + } + }else if (typeof obj === 'bigint'){ + newobj=this._encode(BigInt64Array.from([obj])); + }else if(obj instanceof this._nj.NdArray){ + let dtype=obj.dtype; + dtype=dtype.replace(/float32/,'single').replace(/float64/,'double'); + newobj={_ArrayType_: dtype.toLowerCase() ,_ArraySize_:obj.shape}; + if(this.opt!==undefined && this.opt.hasOwnProperty('compression')){ + newobj._ArrayZipType_=this.opt.compression; + newobj._ArrayZipSize_=[1, obj.size]; + newobj._ArrayZipData_=this.zip(obj.selection.data.buffer,this.opt.compression); + }else{ + newobj._ArrayData_=obj.flatten().tolist(); + } + }else if(obj instanceof Map){ + newobj={_MapData_:[]}; + obj.forEach(function(value, key) { + newobj._MapData_.push([ this._encode(key), this._encode(value)]); + }.bind(this)); + }else if(typeof obj == 'object' && obj !== null){ + for (var k in obj){ + newobj[k]=this._encode(obj[k]); + } + } + return newobj; + } + + _decode(obj){ + let newobj=obj; + if(obj instanceof Array){ + obj.forEach(function(e,idx,orig){ + orig[idx]=this._decode(e); + }.bind(this)); + newobj=obj; + }else if( (typeof obj == 'string') && obj.length<=6 && obj.slice(-1)=='_'){ + if(obj =='_Inf_') + newobj=Infinity; + else if(obj=='-_Inf_') + newobj=-Infinity; + else if(obj=='_NaN_') + newobj=NaN; + }else if(typeof obj == 'object' && obj !== null){ + if(obj.hasOwnProperty('_ArrayType_')){ + let type=obj._ArrayType_; + let data; + type=type.replace(/single/,'float32').replace(/double/,'float64'); + let typename=type.charAt(0).toUpperCase() + type.substring(1) + "Array"; + if(type=='int64' || type=='uint64') + typename='Big'+typename; + if(obj.hasOwnProperty('_ArrayZipData_')){ + data=this.unzip(new Uint8Array(Buffer.from(obj._ArrayZipData_, 'base64')),obj._ArrayZipType_); + //data=Uint8Array.from(data); + if(this.typedfun[typename] == null) + this.typedfun[typename]=new Function('d', 'return new '+typename+'(d)'); + let typecast=this.typedfun[typename]; + data=typecast(data.buffer); + //data=this._nj.array(data,type).reshape(obj._ArraySize_); + }else if(obj.hasOwnProperty('_ArrayData_')){ + data=obj._ArrayData_; + } + newobj=data; + return newobj; + }else if(obj.hasOwnProperty('_MapData_') && Array.isArray(obj._MapData_)){ + newobj=new Map(); + obj._MapData_.forEach(function(e){ + newobj.set(this._decode(e[0]),this._decode(e[1])); + }.bind(this)); + return newobj; + }else if(obj.hasOwnProperty('_ByteStream_')){ + newobj=new Blob(atob(obj._ByteStream_),{type: "octet/stream"}); + return newobj; + }else if(obj.hasOwnProperty('_TableData_') && + obj._TableData_.hasOwnProperty('_TableRecords_') && + obj._TableData_._TableRecords_.length){ + newobj={}; + if(obj._TableData_._TableCols_.length==obj._TableData_._TableRecords_[0].length){ + obj._TableData_._TableCols_.forEach(function(e){ + newobj[e]=[]; + }); + obj._TableRecords_.forEach(function(e){ + for(let i=0;i