Skip to content

Commit

Permalink
call jsonencode/decode in jsave/jload, parse embedded jdata struct
Browse files Browse the repository at this point in the history
  • Loading branch information
@fangq
fangq committed Jun 6, 2020
1 parent 9434103 commit eefccf3
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 70 deletions.
33 changes: 31 additions & 2 deletions jdatadecode.m
View file
Expand Up @@ -4,7 +4,7 @@
%
% Convert all JData object (in the form of a struct array) into an array
% (accepts JData objects loaded from either loadjson/loadubjson or
% jsondecode for MATLAB R2018a or later)
% jsondecode for MATLAB R2016b or later)
%
% This function implements the JData Specification Draft 3 (Jun. 2020)
% see http://github.com/fangq/jdata for details
Expand Down Expand Up @@ -113,6 +113,9 @@
for j=1:len
if(isfield(data,N_('_ArrayZipSize_')) && isfield(data,N_('_ArrayZipData_')))
zipmethod='zip';
if(isstruct(data(j).(N_('_ArrayZipSize_'))))
data(j).(N_('_ArrayZipSize_'))=jdatadecode(data(j).(N_('_ArrayZipSize_')),opt);
end
dims=data(j).(N_('_ArrayZipSize_'))(:)';
if(length(dims)==1)
dims=[1 dims];
Expand Down Expand Up @@ -140,6 +143,9 @@
error('compression method is not supported');
end
else
if(isstruct(data(j).(N_('_ArrayData_'))))
data(j).(N_('_ArrayData_'))=jdatadecode(data(j).(N_('_ArrayData_')),opt);
end
if(isstruct(data(j).(N_('_ArrayData_'))) && isfield(data(j).(N_('_ArrayData_')),N_('_ArrayType_')))
data(j).(N_('_ArrayData_'))=jdatadecode(data(j).(N_('_ArrayData_')),varargin{:});
end
Expand All @@ -149,6 +155,9 @@
ndata=cast(data(j).(N_('_ArrayData_')),char(data(j).(N_('_ArrayType_'))));
end
if(isfield(data,N_('_ArrayZipSize_')))
if(isstruct(data(j).(N_('_ArrayZipSize_'))))
data(j).(N_('_ArrayZipSize_'))=jdatadecode(data(j).(N_('_ArrayZipSize_')),opt);
end
dims=data(j).(N_('_ArrayZipSize_'))(:)';
if(iscell(dims))
dims=cell2mat(dims);
Expand All @@ -160,6 +169,9 @@
ndata=permute(ndata,ndims(ndata):-1:1);
end
iscpx=0;
if(isfield(data,N_('_ArrayIsComplex_')) && isstruct(data(j).(N_('_ArrayIsComplex_'))) )
data(j).(N_('_ArrayIsComplex_'))=jdatadecode(data(j).(N_('_ArrayIsComplex_')),opt);
end
if(isfield(data,N_('_ArrayIsComplex_')) && data(j).(N_('_ArrayIsComplex_')) )
iscpx=1;
end
Expand All @@ -170,8 +182,14 @@
iscol=1;
end
end
if(isfield(data,N_('_ArrayIsSparse_')) && isstruct(data(j).(N_('_ArrayIsSparse_'))) )
data(j).(N_('_ArrayIsSparse_'))=jdatadecode(data(j).(N_('_ArrayIsSparse_')),opt);
end
if(isfield(data,N_('_ArrayIsSparse_')) && data(j).(N_('_ArrayIsSparse_')))
if(isfield(data,N_('_ArraySize_')))
if(isstruct(data(j).(N_('_ArraySize_'))))
data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt);
end
dim=data(j).(N_('_ArraySize_'))(:)';
if(iscell(dim))
dim=cell2mat(dim);
Expand Down Expand Up @@ -203,6 +221,9 @@
ndata=sparse(ndata(1,:),ndata(2,:),ndata(3,:));
end
elseif(isfield(data,N_('_ArrayShape_')))
if(isstruct(data(j).(N_('_ArrayShape_'))))
data(j).(N_('_ArrayShape_'))=jdatadecode(data(j).(N_('_ArrayShape_')),opt);
end
if(iscpx)
if(size(ndata,1)==2)
dim=size(ndata);
Expand All @@ -227,7 +248,11 @@
else
datasize=size(arraydata);
end
if(isstruct(data(j).(N_('_ArraySize_'))))
data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt);
end
arraysize=data.(N_('_ArraySize_'));

if(iscell(arraysize))
arraysize=cell2mat(arraysize);
end
Expand Down Expand Up @@ -278,12 +303,16 @@
end
ndata=spdiags(reshape(arraydata,min(arraysize),datasize(1)),double(shapeid{2}):-1:-double(shapeid{3}),arraysize(1),arraysize(2));
elseif(strcmpi(shapeid{1},'toeplitz'))
ndata=toeplitz(arraydata(:,1),arraydata(:,2)).';
arraydata=reshape(arraydata,flipud(datasize(:))');
ndata=toeplitz(arraydata(1:arraysize(1),2),arraydata(1:arraysize(2),1));
end
if(opt.fullarrayshape && issparse(ndata))
ndata=cast(full(ndata),data(j).(N_('_ArrayType_')));
end
elseif(isfield(data,N_('_ArraySize_')))
if(isstruct(data(j).(N_('_ArraySize_'))))
data(j).(N_('_ArraySize_'))=jdatadecode(data(j).(N_('_ArraySize_')),opt);
end
if(iscpx)
ndata=complex(ndata(1,:),ndata(2,:));
end
Expand Down
143 changes: 83 additions & 60 deletions jdataencode.m
View file
Expand Up @@ -5,8 +5,11 @@
% jdata=jdataencode(data, options)
% jdata=jdataencode(data, 'Param1',value1, 'Param2',value2,...)
%
% Serialize a MATLAB struct or cell array into a JData-compliant
% structure as defined in the JData spec: http://github.com/fangq/jdata
% Annotate a MATLAB struct or cell array into a JData-compliant data
% structure as defined in the JData spec: http://github.com/fangq/jdata.
% This encoded form servers as an intermediate format that allows unambiguous
% storage, exchange of complex data structures and easy-to-serialize by
% json encoders such as savejson and jsonencode (MATLAB R2016b or newer)
%
% This function implements the JData Specification Draft 3 (Jun. 2020)
% see http://github.com/fangq/jdata for details
Expand All @@ -17,6 +20,11 @@
% data: a structure (array) or cell (array) to be encoded.
% options: (optional) a struct or Param/value pairs for user
% specified options (first in [.|.] is the default)
% AnnotateArray: [0|1] - if set to 1, convert all 1D/2D matrices
% to the annotated JData array format to preserve data types;
% N-D (N>2), complex and sparse arrays are encoded using the
% annotated format by default. Please set this option to 1 if
% you intend to use MATLAB's jsonencode to convert to JSON.
% Base64: [0|1] if set to 1, _ArrayZipData_ is assumed to
% be encoded with base64 format and need to be
% decoded first. This is needed for JSON but not
Expand Down Expand Up @@ -50,6 +58,14 @@
% example:
% jd=jdataencode(struct('a',rand(5)+1i*rand(5),'b',[],'c',sparse(5,5)))
%
% encodedmat=jdataencode(single(magic(5)),'annotatearray',1,'prefix','x')
% jdatadecode(jsondecode(jsonencode(encodedmat))) % serialize by jsonencode
% jdatadecode(loadjson(savejson('',encodedmat))) % serialize by savejson
%
% encodedtoeplitz=jdataencode(uint8(toeplitz([1,2,3,4],[1,5,6])),'usearrayshape',1,'prefix','x')
% jdatadecode(jsondecode(jsonencode(encodedtoeplitz))) % serialize by jsonencode
% jdatadecode(loadjson(savejson('',encodedtoeplitz))) % serialize by savejson
%
% license:
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
%
Expand Down Expand Up @@ -77,6 +93,7 @@
opt.usearrayzipsize=jsonopt('UseArrayZipSize',1,opt);
opt.messagepack=jsonopt('MessagePack',0,opt);
opt.usearrayshape=jsonopt('UseArrayShape',0,opt) && exist('bandwidth');
opt.annotatearray=jsonopt('AnnotateArray',0,opt);

jdata=obj2jd(data,opt);

Expand Down Expand Up @@ -166,14 +183,9 @@
zipmethod=varargin{1}.compression;
minsize=varargin{1}.compressarraysize;

% no encoding for char arrays or non-sparse real vectors
if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ...
((isvector(item) || (ndims(item)==2 && ~varargin{1}.usearrayshape)) ...
&& isreal(item) && ~issparse(item)))
newitem=item;
return;
% 2d numerical (real/complex/sparse) arrays with _ArrayShape_ encoding enabled
elseif(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
if(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
encoded=1;
if(~isreal(item))
newitem.(N('_ArrayIsComplex_'))=true;
end
Expand Down Expand Up @@ -214,11 +226,12 @@
newitem.(N('_ArrayData_'))(1,1:size(item,2))=item(1,:);
newitem.(N('_ArrayData_'))(2,1:size(item,1))=item(:,1).';
else % full matrix
newitem=item;
newitem=rmfield(newitem,N('_ArrayZipSize_'));
encoded=0;
end

% serialize complex data at last
if(isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_'))))
if(encoded && isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_'))))
item=squeeze(zeros([2, size(newitem.(N('_ArrayData_')))]));
item(1,:)=real(newitem.(N('_ArrayData_'))(:));
item(2,:)=imag(newitem.(N('_ArrayData_'))(:));
Expand All @@ -228,64 +241,74 @@

% wrap _ArrayData_ into a single row vector, and store preprocessed
% size to _ArrayZipSize_ (force varargin{1}.usearrayzipsize=true)
if(isstruct(newitem) && ~isvector(newitem.(N('_ArrayData_'))))
item=newitem.(N('_ArrayData_'));
item=permute(item,ndims(item):-1:1);
newitem.(N('_ArrayData_'))=item(:).';
else
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end

newitem.(N('_ArrayData_'))=full(newitem.(N('_ArrayData_')));
else
if(isa(item,'logical'))
item=uint8(item);
if(encoded)
if(isstruct(newitem) && ~isvector(newitem.(N('_ArrayData_'))))
item=newitem.(N('_ArrayData_'));
item=permute(item,ndims(item):-1:1);
newitem.(N('_ArrayData_'))=item(:).';
else
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end
newitem.(N('_ArrayData_'))=full(newitem.(N('_ArrayData_')));
return
end
end

if(isreal(item))
if(issparse(item))
fulldata=full(item(find(item)));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[2+(~isvector(item)),length(fulldata)];
if(isvector(item))
newitem.(N('_ArrayData_'))=[find(item(:))', fulldata(:)'];
else
[ix,iy]=find(item);
newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)', fulldata(:)'];
end
% no encoding for char arrays or non-sparse real vectors
if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ...
((isvector(item) || ndims(item)==2) && isreal(item) && ~issparse(item) && ...
~varargin{1}.annotatearray))
newitem=item;
return;
end

if(isa(item,'logical'))
item=uint8(item);
end

if(isreal(item))
if(issparse(item))
fulldata=full(item(item~=0));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[2+(~isvector(item)),length(fulldata)];
if(isvector(item))
newitem.(N('_ArrayData_'))=[find(item(:))', fulldata(:)'];
else
if(varargin{1}.formatversion>1.9)
item=permute(item,ndims(item):-1:1);
end
newitem.(N('_ArrayData_'))=item(:)';
[ix,iy]=find(item);
newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)', fulldata(:)'];
end
else
newitem.(N('_ArrayIsComplex_'))=true;
if(issparse(item))
fulldata=full(item(find(item)));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[3+(~isvector(item)),length(fulldata)];
if(isvector(item))
newitem.(N('_ArrayData_'))=[find(item(:))', real(fulldata(:))', imag(fulldata(:))'];
else
[ix,iy]=find(item);
newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)' , real(fulldata(:))', imag(fulldata(:))'];
end
if(varargin{1}.formatversion>1.9)
item=permute(item,ndims(item):-1:1);
end
newitem.(N('_ArrayData_'))=item(:)';
end
else
newitem.(N('_ArrayIsComplex_'))=true;
if(issparse(item))
fulldata=full(item(item~=0));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[3+(~isvector(item)),length(fulldata)];
if(isvector(item))
newitem.(N('_ArrayData_'))=[find(item(:))', real(fulldata(:))', imag(fulldata(:))'];
else
if(varargin{1}.formatversion>1.9)
item=permute(item,ndims(item):-1:1);
end
newitem.(N('_ArrayZipSize_'))=[2,numel(item)];
newitem.(N('_ArrayData_'))=[real(item(:))', imag(item(:))'];
[ix,iy]=find(item);
newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)' , real(fulldata(:))', imag(fulldata(:))'];
end
else
if(varargin{1}.formatversion>1.9)
item=permute(item,ndims(item):-1:1);
end
newitem.(N('_ArrayZipSize_'))=[2,numel(item)];
newitem.(N('_ArrayData_'))=[real(item(:))', imag(item(:))'];
end
end

if(varargin{1}.usearrayzipsize==0 && isfield(newitem,N('_ArrayZipSize_')))
data=newitem.(N('_ArrayData_'));
data=reshape(data,fliplr(newitem.(N('_ArrayZipSize_'))));
newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1);
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end
if(varargin{1}.usearrayzipsize==0 && isfield(newitem,N('_ArrayZipSize_')))
data=newitem.(N('_ArrayData_'));
data=reshape(data,fliplr(newitem.(N('_ArrayZipSize_'))));
newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1);
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end

if(~isempty(zipmethod) && numel(item)>minsize)
Expand Down
22 changes: 19 additions & 3 deletions jload.m
View file
Expand Up @@ -23,8 +23,11 @@
% ws ['base'|'wsname']: the name of the workspace in which the
% variables are to be saved
% vars [{'var1','var2',...}]: list of variables to be saved
% Header [0|1]: if set to 1, return the metadata of the variables
% header [0|1]: if set to 1, return the metadata of the variables
% stored in the file
% matlab [0|1] if set to 1, use matlab's built-in jsondecode to
% parse the json file and then decode the output by
% jdatadecode; input file must have a suffix of .jdt
%
% all options for loadubjson/loadjson (depends on file suffix)
% can be used to adjust the parsing options
Expand Down Expand Up @@ -61,7 +64,16 @@
loadfun=@loadmsgpack;
end

header=loadfun(filename,'ObjectID',1, varargin{:});
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
jsonstr=fileread(filename);
pos=regexp(jsonstr,'}\n\n\n{"WorkspaceData":','once');
if(isempty(pos))
error('the json file is not generated using matlab''s jsonencode');
end
header=jsondecode(jsonstr(1:pos+1));
else
header=loadfun(filename,'ObjectID',1, varargin{:});
end

if(jsonopt('Header',0,opt))
varargout{1}=header;
Expand All @@ -78,7 +90,11 @@
error('specified variable is not found');
end

body=loadfun(filename,'ObjectID',2, varargin{:});
if(jsonopt('matlab',0,opt) && exist('jsonencode','builtin'))
body=jdatadecode(jsondecode(jsonstr(pos+4:end)));
else
body=jsondecode(filename,'ObjectID',2, varargin{:});
end

for i=1:length(varlist)
assignin(ws, varlist{i}, body.WorkspaceData.(varlist{i}));
Expand Down

0 comments on commit eefccf3

Please sign in to comment.