Skip to content

Commit

Permalink
support lzma and lzip compression decompression via zmat toolbox (htt…
Browse files Browse the repository at this point in the history
  • Loading branch information
fangq committed May 31, 2019
1 parent 599ee4c commit 0c467ee
Show file tree
Hide file tree
Showing 7 changed files with 110 additions and 43 deletions.
23 changes: 23 additions & 0 deletions lzipdecode.m
@@ -0,0 +1,23 @@
function output = lzipdecode(input)
%LZIPDECODE Decompress input bytes using lzip.
%
% output = lzipdecode(input)
%
% The function takes a compressed byte array INPUT and returns inflated
% bytes OUTPUT. The INPUT is a result of LZIPDECODE function. The OUTPUT
% is always an 1-by-N uint8 array.
%
% See also lzipencode typecast
%
% License : BSD, see LICENSE_*.txt
%

if(nargin==0)
error('you must provide at least 1 input');
end
if(exist('zmat')==3)
output=zmat(uint8(input),0,'lzip');
return;
else
error('you must install ZMat toolbox to use this feature: http://github.com/fangq/zmat_mex')
end
23 changes: 23 additions & 0 deletions lzipencode.m
@@ -0,0 +1,23 @@
function output = lzipencode(input)
%LZIPENCODE Compress input bytes with lzip.
%
% output = lzipencode(input)
%
% The function takes a char, int8, or uint8 array INPUT and returns
% compressed bytes OUTPUT as a uint8 array. Note that the compression
% doesn't preserve input dimensions.
%
% See also lzipdecode
%
% License : BSD, see LICENSE_*.txt
%

if(nargin==0)
error('you must provide at least 1 input');
end
if(exist('zmat')==3)
output=zmat(uint8(input),1,'lzip');
return;
else
error('you must install ZMat toolbox to use this feature: http://github.com/fangq/zmat_mex')
end
23 changes: 23 additions & 0 deletions lzmadecode.m
@@ -0,0 +1,23 @@
function output = lzmadecode(input)
%LZMADECODE Decompress input bytes using lzma.
%
% output = lzmadecode(input)
%
% The function takes a compressed byte array INPUT and returns inflated
% bytes OUTPUT. The INPUT is a result of LZMADECODE function. The OUTPUT
% is always an 1-by-N uint8 array.
%
% See also lzmaencode typecast
%
% License : BSD, see LICENSE_*.txt
%

if(nargin==0)
error('you must provide at least 1 input');
end
if(exist('zmat')==3)
output=zmat(uint8(input),0,'lzma');
return;
else
error('you must install ZMat toolbox to use this feature: http://github.com/fangq/zmat_mex')
end
23 changes: 23 additions & 0 deletions lzmaencode.m
@@ -0,0 +1,23 @@
function output = lzmaencode(input)
%LZMAENCODE Compress input bytes with lzma.
%
% output = lzmaencode(input)
%
% The function takes a char, int8, or uint8 array INPUT and returns
% compressed bytes OUTPUT as a uint8 array. Note that the compression
% doesn't preserve input dimensions.
%
% See also lzmadecode
%
% License : BSD, see LICENSE_*.txt
%

if(nargin==0)
error('you must provide at least 1 input');
end
if(exist('zmat')==3)
output=zmat(uint8(input),1,'lzma');
return;
else
error('you must install ZMat toolbox to use this feature: http://github.com/fangq/zmat_mex')
end
24 changes: 7 additions & 17 deletions savejson.m
Expand Up @@ -67,8 +67,8 @@
% back to the string form
% opt.SaveBinary [0|1]: 1 - save the JSON file in binary mode; 0 - text mode.
% opt.Compact [0|1]: 1- out compact JSON format (remove all newlines and tabs)
% opt.Compression 'zlib' or 'gzip': specify array compression
% method; currently only supports 'gzip' or 'zlib'. The
% opt.Compression 'zlib', 'gzip', 'lzma' or 'lzip': specify array compression
% method; currently only supports 4 methods. The
% data compression only applicable to numerical arrays
% in 3D or higher dimensions, or when ArrayToStruct
% is 1 for 1D or 2D arrays. If one wants to
Expand Down Expand Up @@ -123,7 +123,7 @@

dozip=jsonopt('Compression','',opt);
if(~isempty(dozip))
if(~(strcmpi(dozip,'gzip') || strcmpi(dozip,'zlib')))
if(isempty(strmatch(dozip,{'zlib','gzip','lzma','lzip'})))
error('compression method "%s" is not supported',dozip);
end
if(exist('zmat')~=3)
Expand Down Expand Up @@ -520,13 +520,8 @@
end
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionSize_": ',regexprep(mat2str(size(fulldata)),'\s+',','), sep);
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionMethod_": "',dozip, ['"' sep]);
if(strcmpi(dozip,'gzip'))
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(gzipencode(typecast(fulldata(:),'uint8'))),['"' nl]);
elseif(strcmpi(dozip,'zlib'))
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(zlibencode(typecast(fulldata(:),'uint8'))),['"' nl]);
else
error('compression method not supported');
end
compfun=str2func([dozip 'encode']);
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(compfun(typecast(fulldata(:),'uint8'))),['"' nl]);
else
if(size(item,1)==1)
% Row vector, store only column indices.
Expand Down Expand Up @@ -554,13 +549,8 @@
end
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionSize_": ',regexprep(mat2str(size(fulldata)),'\s+',','), sep);
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressionMethod_": "',dozip, ['"' sep]);
if(strcmpi(dozip,'gzip'))
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(gzipencode(typecast(fulldata(:),'uint8'))),['"' nl]);
elseif(strcmpi(dozip,'zlib'))
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(zlibencode(typecast(fulldata(:),'uint8'))),['"' nl]);
else
error('compression method not supported');
end
compfun=str2func([dozip 'encode']);
txt=sprintf(dataformat,txt,padding0,'"_ArrayCompressedData_": "',base64encode(compfun(typecast(fulldata(:),'uint8'))),['"' nl]);
else
if(isreal(item))
txt=sprintf(dataformat,txt,padding0,'"_ArrayData_": ',...
Expand Down
24 changes: 7 additions & 17 deletions saveubjson.m
Expand Up @@ -61,8 +61,8 @@
% wrapped inside a function call as 'foo(...);'
% opt.UnpackHex [1|0]: conver the 0x[hex code] output by loadjson
% back to the string form
% opt.Compression 'zlib' or 'gzip': specify array compression
% method; currently only supports 'gzip' or 'zlib'. The
% opt.Compression 'zlib', 'gzip', 'lzma' or 'lzip': specify array compression
% method; currently only supports 4 methods. The
% data compression only applicable to numerical arrays
% in 3D or higher dimensions, or when ArrayToStruct
% is 1 for 1D or 2D arrays. If one wants to
Expand Down Expand Up @@ -123,7 +123,7 @@

dozip=jsonopt('Compression','',opt);
if(~isempty(dozip))
if(~(strcmpi(dozip,'gzip') || strcmpi(dozip,'zlib')))
if(isempty(strmatch(dozip,{'zlib','gzip','lzma','lzip'})))
error('compression method "%s" is not supported',dozip);
end
if(exist('zmat')~=3)
Expand Down Expand Up @@ -491,13 +491,8 @@
cid=I_(uint32(max(size(fulldata))));
txt=[txt, N_('_ArrayCompressionSize_'),I_a(size(fulldata),cid(1),Imarker)];
txt=[txt, N_('_ArrayCompressionMethod_'),S_(dozip)];
if(strcmpi(dozip,'gzip'))
txt=[txt,N_('_ArrayCompressedData_'), I_a(gzipencode(typecast(fulldata(:),'uint8')),Imarker(1),Imarker)];
elseif(strcmpi(dozip,'zlib'))
txt=[txt,N_('_ArrayCompressedData_'), I_a(zlibencode(typecast(fulldata(:),'uint8')),Imarker(1),Imarker)];
else
error('compression method not supported');
end
compfun=str2func([dozip 'encode']);
txt=[txt,N_('_ArrayCompressedData_'), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),Imarker)];
childcount=childcount+3;
else
if(size(item,1)==1)
Expand Down Expand Up @@ -529,13 +524,8 @@
cid=I_(uint32(max(size(fulldata))));
txt=[txt, N_('_ArrayCompressionSize_'),I_a(size(fulldata),cid(1),Imarker)];
txt=[txt, N_('_ArrayCompressionMethod_'),S_(dozip)];
if(strcmpi(dozip,'gzip'))
txt=[txt,N_('_ArrayCompressedData_'), I_a(gzipencode(typecast(fulldata(:),'uint8')),Imarker(1),Imarker)];
elseif(strcmpi(dozip,'zlib'))
txt=[txt,N_('_ArrayCompressedData_'), I_a(zlibencode(typecast(fulldata(:),'uint8')),Imarker(1),Imarker)];
else
error('compression method not supported');
end
compfun=str2func([dozip 'encode']);
txt=[txt,N_('_ArrayCompressedData_'), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),Imarker)];
childcount=childcount+3;
else
if(isreal(item))
Expand Down
13 changes: 4 additions & 9 deletions struct2jdata.m
Expand Up @@ -68,17 +68,12 @@
if(~isempty(strmatch('x0x5F_ArrayCompressionMethod_',fn)))
zipmethod=data(j).x0x5F_ArrayCompressionMethod_;
end
if(strcmpi(zipmethod,'gzip'))
if(~isempty(strmatch(zipmethod,{'zlib','gzip','lzma','lzip'})))
decompfun=str2func([zipmethod 'decode']);
if(needbase64)
ndata=reshape(typecast(gzipdecode(base64decode(data(j).x0x5F_ArrayCompressedData_)),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
ndata=reshape(typecast(decompfun(base64decode(data(j).x0x5F_ArrayCompressedData_)),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
else
ndata=reshape(typecast(gzipdecode(data(j).x0x5F_ArrayCompressedData_),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
end
elseif(strcmpi(zipmethod,'zlib'))
if(needbase64)
ndata=reshape(typecast(zlibdecode(base64decode(data(j).x0x5F_ArrayCompressedData_)),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
else
ndata=reshape(typecast(zlibdecode(data(j).x0x5F_ArrayCompressedData_),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
ndata=reshape(typecast(decompfun(data(j).x0x5F_ArrayCompressedData_),data(j).x0x5F_ArrayType_),data(j).x0x5F_ArrayCompressionSize_);
end
else
error('compression method is not supported');
Expand Down

0 comments on commit 0c467ee

Please sign in to comment.