Permalink
Fetching contributors…
Cannot retrieve contributors at this time
1023 lines (903 sloc) 48.3 KB
% Simulation disk logger.
% Stores and retrieves simulated dynamics and simulation meta data
% to/from disk. Generally, each batch of simulations is stored as folder
% containing 1 subfolder for each simulation. Each simulation subfolder
% contains the simulated dynamics of a single cell. The simulated
% dynamics of each single cell are stored in two ways:
% 1) Simulated dynamics are stored indexed primarily by time with the
% file name pattern state-(\d+).mat
% 2) Simulated dynamics are stored indexed primarily by state with the
% file name pattern state-(state name)-(property name).mat
% During the execution of each simulation results are stored in the first
% form. However this form is inefficient for most analyses. Consequently,
% after the completion of each simulation, simulated dynamics are
% reindexed by state.
%
% The SimuationEnsemble class provides additional methods for retrieving
% the simulated dynamics of multiple cells and entire cellular
% populations. The SimulationDiskUtil class provides additional methods
% for retrieving simulation meta data.
%
% Author: Jonathan Karr, jkarr@stanford.edu
% Affilitation: Covert Lab, Department of Bioengineering, Stanford University
% Last updated: 1/10/2011
classdef DiskLogger < edu.stanford.covert.cell.sim.util.Logger
%options
properties (SetAccess = protected)
metadata %metadata
outputDirectory %output directory
segmentSizeStep %max state history to cache in memory during a run (s)
verbosity %verbosity
end
%saved data
properties (SetAccess = protected)
log
randStreamStates
end
%indices into simulation state
properties (Access = protected)
stateIndex_time
end
methods
function this = DiskLogger(outputDirectory, segmentSizeStep, metadata, verbosity)
if ~exist(outputDirectory, 'dir')
mkdir(outputDirectory);
end
if ~exist('metadata', 'var')
metadata = struct;
end
if ~exist('verbosity', 'var')
verbosity = 0;
end
this.outputDirectory = outputDirectory;
this.segmentSizeStep = segmentSizeStep;
this.metadata = metadata;
this.verbosity = verbosity;
end
end
methods
function setOptions(this, varargin)
if isstruct(varargin{1})
options = varargin{1}; %#ok<*PROP>
else
options = struct(varargin{:});
end
metaClass = metaclass(this);
fields = intersect(fieldnames(options), cellfun(@(x) x.Name, metaClass.Properties, 'UniformOutput',false));
for i = 1:numel(fields)
this.(fields{i}) = options.(fields{i});
end
end
function this = addMetadata(this, varargin)
if isstruct(varargin{1})
metadata = varargin{1}; %#ok<*PROP>
else
metadata = struct(varargin{:});
end
names = fieldnames(metadata);
for i = 1:length(names)
this.metadata.(names{i}) = metadata.(names{i});
end
end
function this = initialize(this, sim)
%% validate options
%segment length
validateattributes(sim.lengthSec / sim.stepSizeSec / this.segmentSizeStep, {'numeric'}, {'real', 'nonnegative', 'integer'});
%downsample step
validateattributes(sim.lengthSec / sim.stepSizeSec / this.segmentSizeStep, {'numeric'}, {'real', 'nonnegative', 'integer'});
%metadata
this.validateMetadata();
%% indices
this.stateIndex_time = sim.stateIndex('Time');
%% metdata
%states
this.metadata.stateNames = cell(0, 2);
this.metadata.dependentStateNames = cell(0, 2);
for j = 1:length(sim.states)
state = sim.states{j};
stateID = {state.wholeCellModelID(7:end)};
this.metadata.stateNames = [
this.metadata.stateNames
repmat(stateID, numel(state.stateNames), 1) state.stateNames
];
this.metadata.dependentStateNames = [
this.metadata.dependentStateNames
repmat(stateID, numel(state.dependentStateNames), 1) state.dependentStateNames
];
end
%start time
this.metadata.startTime = datestr(clock, 'yyyy-mm-dd HH:MM:SS');
this.metadata.endTime = [];
this.metadata.lengthSec = [];
%output directory
this.metadata.outputDirectory = strrep(this.outputDirectory, '\', '/');
if ~isempty(this.outputDirectory) && ~exist(this.outputDirectory, 'dir')
mkdir(this.outputDirectory);
end
%segment step
this.metadata.segmentSizeStep = this.segmentSizeStep;
%down sample step
this.metadata.downsampleStepSec = sim.stepSizeSec;
%% store initial state as "segment zero"
metaStates = this.getMetaStates(sim, true);
this.log = this.allocateMemory(metaStates, 1);
this.copyFromState(sim, 1);
this.saveSegmentToDisk(0);
%% allocate memory
%cell state
this.log = this.allocateMemory(metaStates, this.segmentSizeStep);
%rand stream states
this.randStreamStates = struct('simulation', [], 'states', struct(), 'processes', struct());
tmp = sim.getRandStreamStates();
this.randStreamStates.simulation = zeros(numel(tmp.simulation), sim.lengthSec + 1);
for i = 1:numel(sim.states)
o = sim.states{i};
this.randStreamStates.states.(o.wholeCellModelID(7:end)) = zeros(numel(tmp.states.(o.wholeCellModelID(7:end))), sim.lengthSec + 1);
end
for i = 1:numel(sim.processes)
o = sim.processes{i};
this.randStreamStates.processes.(o.wholeCellModelID(9:end)) = zeros(numel(tmp.processes.(o.wholeCellModelID(9:end))), sim.lengthSec + 1);
end
this.copyRandStreamStates(sim);
end
function this = append(this, sim)
%cell state
nSteps = sim.states{this.stateIndex_time}.values / sim.stepSizeSec;
i = mod(nSteps - 1, this.segmentSizeStep) + 1;
this.copyFromState(sim, i);
if i == this.segmentSizeStep
this.saveSegmentToDisk(nSteps / this.segmentSizeStep);
end
%rand stream states
this.copyRandStreamStates(sim);
end
function this = finalize(this, sim)
%references
states = sim.states;
%meta data -- end time
this.metadata.endTime = datestr(clock, 'yyyy-mm-dd HH:MM:SS');
this.metadata.lengthSec = sim.states{this.stateIndex_time}.values;
%append last data bits, trim last segment, save, and clear
nSteps = sim.states{this.stateIndex_time}.values / sim.stepSizeSec;
i = mod(nSteps - 1, this.segmentSizeStep) + 1;
this.copyFromState(sim, i);
for j = 1:length(states)
state = states{j};
stateID = state.wholeCellModelID(7:end);
names = [state.stateNames; state.dependentStateNames];
for k = 1:length(names)
name = names{k};
this.log.(stateID).(name) = this.log.(stateID).(name)(:, :, 1:i);
end
end
this.saveSegmentToDisk(ceil(nSteps / this.segmentSizeStep));
%append rand stream states, trim, and save
this.copyRandStreamStates(sim);
this.randStreamStates.simulation = this.randStreamStates.simulation(:, 1:nSteps+1);
for i = 1:numel(sim.states)
o = sim.states{i};
this.randStreamStates.states.(o.wholeCellModelID(7:end)) = this.randStreamStates.states.(o.wholeCellModelID(7:end))(:, 1:nSteps+1);
end
for i = 1:numel(sim.processes)
o = sim.processes{i};
this.randStreamStates.processes.(o.wholeCellModelID(9:end)) = this.randStreamStates.processes.(o.wholeCellModelID(9:end))(:, 1:nSteps+1);
end
%store to disk
this.saveMetadata(sim);
this.saveOptions(sim);
this.saveParameters(sim);
this.saveFittedConstants(sim);
this.saveRandStreamStates(sim);
end
end
methods
function validateMetadata(this)
if ...
~isfield(this.metadata, 'shortDescription') || ...
~isfield(this.metadata, 'longDescription') || ...
~isfield(this.metadata, 'email') || ...
~isfield(this.metadata, 'firstName') || ...
~isfield(this.metadata, 'lastName') || ...
~isfield(this.metadata, 'affiliation') || ...
~isfield(this.metadata, 'knowledgeBaseWID') || ...
~isfield(this.metadata, 'revision') || ...
~isfield(this.metadata, 'differencesFromRevision') || ...
~isfield(this.metadata, 'userName') || ...
~isfield(this.metadata, 'hostName') || ...
~isfield(this.metadata, 'ipAddress')
throw(MException('DiskLogger:invalidMetadata', 'Metadata missing'));
end
end
%Copies the current state to a particular index in a segment.
function copyFromState(this, sim, k)
import edu.stanford.covert.cell.sim.util.DiskLogger;
states = sim.states;
for i = 1:length(states)
state = states{i};
stateID = state.wholeCellModelID(7:end);
names = [state.stateNames; state.dependentStateNames];
for j = 1:length(names)
name = names{j};
data = state.(name);
if size(data, 1) > size(this.log.(stateID).(name), 1) || ...
size(data, 2) > size(this.log.(stateID).(name), 2)
tmp = this.log.(stateID).(name);
this.log.(stateID).(name) = DiskLogger.allocateData(class(tmp), [size(data) size(tmp, 3)]);
this.log.(stateID).(name)(1:size(tmp,1), 1:size(tmp, 2), :) = tmp;
end
if isa(data, 'edu.stanford.covert.util.SparseMat')
if k == 1
this.log.(stateID).(name) = data;
else
this.log.(stateID).(name) = cat(3, this.log.(stateID).(name), data);
end
else
this.log.(stateID).(name)(1:size(data, 1), 1:size(data, 2), k) = data;
end
end
end
end
function copyRandStreamStates(this, sim)
nSteps = sim.states{this.stateIndex_time}.values / sim.stepSizeSec;
tmp = sim.getRandStreamStates();
this.randStreamStates.simulation(:, nSteps + 1) = tmp.simulation;
for i = 1:numel(sim.states)
o = sim.states{i};
this.randStreamStates.states.(o.wholeCellModelID(7:end))(:, nSteps + 1) = tmp.states.(o.wholeCellModelID(7:end));
end
for i = 1:numel(sim.processes)
o = sim.processes{i};
this.randStreamStates.processes.(o.wholeCellModelID(9:end))(:, nSteps + 1) = tmp.processes.(o.wholeCellModelID(9:end));
end
end
function saveMetadata(this, ~)
this.saveStructToDisk(this.outputDirectory, 'metadata.mat', this.metadata);
end
function saveOptions(this, sim)
this.saveStructToDisk(this.outputDirectory, 'options.mat', sim.getOptions);
end
function saveParameters(this, sim)
this.saveStructToDisk(this.outputDirectory, 'parameters.mat', sim.getParameters);
end
function saveFittedConstants(this, sim)
this.saveStructToDisk(this.outputDirectory, 'fittedConstants.mat', sim.getFittedConstants);
end
function saveRandStreamStates(this, ~)
this.saveStructToDisk(this.outputDirectory, 'randStreamStates.mat', this.randStreamStates);
end
function saveSegmentToDisk(this, segmentIdx)
this.saveStructToDisk(this.outputDirectory, ['state-' num2str(segmentIdx) '.mat'], this.log);
end
%number of time steps per segment
function value = getSegmentLength(this, sim)
value = this.segmentSizeSec / sim.stepSizeSec;
end
%number of time segments
function value = getNumSegments(this, sim)
value = sim.lengthSec / this.getSegmentLength;
end
function clearLog(this)
this.log = [];
end
end
%load from disk
methods (Static)
function metaStates = getMetaStates(sim, includeDependentStates)
metaStates.names = cell(0, 1);
metaStates.properties = cell(0, 4);
for i = 1:length(sim.states)
state = sim.states{i};
stateID = strrep(state.wholeCellModelID, 'State_', '');
metaStates.names = [
metaStates.names;
stateID];
names = state.stateNames;
if nargin >= 2 && exist('includeDependentStates', 'var') && includeDependentStates
names = [names; state.dependentStateNames]; %#ok<AGROW>
end
for j = 1:length(names)
metaStates.properties = [
metaStates.properties;
{stateID names{j} class(state.(names{j})) size(state.(names{j}))}];
end
end
end
function [stateNames, stateDataTypes, isStateDataBuiltinDenseMatrix] = getAvailableStates(outputDirectory)
import edu.stanford.covert.cell.sim.util.DiskLogger;
log0 = DiskLogger.loadStructFromDisk(sprintf('%s/state-%d.mat', outputDirectory, 0));
stateNames = cell(0, 2);
stateDataTypes = cell(0, 1);
isStateDataBuiltinDenseMatrix = false(0, 1);
fields = fieldnames(log0);
for i = 1:numel(fields)
fieldsi = fieldnames(log0.(fields{i}));
stateNames = [
stateNames;
repmat(fields(i), size(fieldsi, 1), 1) fieldsi(:)]; %#ok<AGROW>
for j = 1:numel(fieldsi)
stateDataTypes = [
stateDataTypes;
class(log0.(fields{i}).(fieldsi{j}))
]; %#ok<AGROW>
isStateDataBuiltinDenseMatrix = [
isStateDataBuiltinDenseMatrix;
isnumeric(log0.(fields{i}).(fieldsi{j})) && ~issparse(log0.(fields{i}).(fieldsi{j}))
]; %#ok<AGROW>
end
end
end
%load simulation from disk
%- stateNames is a n x 2 cell array; 1st col->state ID, 2nd col->property name
%- downsampleType can be 'extract' or 'mean'
function [states, metadata, options, parameters, fittedConstants, randStreamStates] = load(...
outputDirectory, stateNames, ...
initTime, finTime, downsampleStepSec, ...
downsampleType)
import edu.stanford.covert.cell.sim.util.DiskLogger;
metadata = DiskLogger.loadMetadata(outputDirectory); metadata.downsampleStepSec = downsampleStepSec;
options = DiskLogger.loadOptions(outputDirectory);
if nargout >= 4
parameters = DiskLogger.loadParameters(outputDirectory);
end
if nargout >= 5
fittedConstants = DiskLogger.loadFittedConstants(outputDirectory);
end
if nargout >= 6
randStreamStates = DiskLogger.loadRandStreamStates(outputDirectory);
end
states = DiskLogger.loadTimecourses(outputDirectory, stateNames, initTime, finTime, downsampleStepSec, downsampleType, options, metadata);
end
function value = loadMetadata(outputDirectory, varargin)
import edu.stanford.covert.cell.sim.util.DiskLogger;
value = DiskLogger.loadStructFromDisk([outputDirectory '/metadata.mat'], varargin{:});
end
function value = loadOptions(outputDirectory, varargin)
import edu.stanford.covert.cell.sim.util.DiskLogger;
value = DiskLogger.loadStructFromDisk([outputDirectory '/options.mat'], varargin{:});
end
function value = loadParameters(outputDirectory, varargin)
import edu.stanford.covert.cell.sim.util.DiskLogger;
value = DiskLogger.loadStructFromDisk([outputDirectory '/parameters.mat'], varargin{:});
end
function value = loadFittedConstants(outputDirectory, varargin)
import edu.stanford.covert.cell.sim.util.DiskLogger;
value = DiskLogger.loadStructFromDisk([outputDirectory '/fittedConstants.mat'], varargin{:});
end
function value = loadRandStreamStates(outputDirectory, varargin)
import edu.stanford.covert.cell.sim.util.DiskLogger;
value = DiskLogger.loadStructFromDisk([outputDirectory '/randStreamStates.mat'], varargin{:});
end
function states = loadTimecourses(outputDirectory, stateNames, initTime, finTime, downsampleStepSec, ...
downsampleType, options, metadata, method)
import edu.stanford.covert.cell.sim.util.DiskLogger;
%process options
if ~exist('downsampleStepSec', 'var') || isempty(downsampleStepSec)
downsampleStepSec = options.stepSizeSec;
elseif mod(downsampleStepSec, options.stepSizeSec) ~= 0
throw(MException('DiskLogger:invalid', 'downsampleStepSec must be a multiple of stepSizeSec'));
end
if ~exist('downsampleType', 'var') || isempty(downsampleType)
downsampleType = 'extract';
elseif ~ismember(downsampleType, {'extract', 'mean'})
throw(MException('DiskLogger:invalid', 'downsampleType must be one of ''extract'' or ''mean'''));
end
if strcmp(downsampleType, 'mean') && downsampleStepSec == options.stepSizeSec
downsampleType = 'extract';
end
if isempty(initTime)
initTime = 1;
else
validateattributes(initTime, {'numeric'}, {'>=', 1, '<=', metadata.lengthSec});
end
if isempty(finTime)
finTime = metadata.lengthSec;
else
validateattributes(finTime, {'numeric'}, {'>=', initTime, '<=', metadata.lengthSec});
end
if ~exist('method', 'var')
method = 'byState';
end
%allocate memory
numSegments = ceil(metadata.lengthSec / options.stepSizeSec / metadata.segmentSizeStep);
if ischar(stateNames) && stateNames(1) == '-'
logN = DiskLogger.loadStructFromDisk(sprintf('%s/state-%d.mat', outputDirectory, numSegments));
metaStates.names = fieldnames(logN);
switch stateNames
case '-all'
if isfield(metadata, 'stateNames')
stateNames = [metadata.stateNames; metadata.dependentStateNames];
else
%for backwards compatability with data from before dependent
%states were stored (Revision < 1733)
stateNames = DiskLogger.getAvailableStates(outputDirectory);
end
case '-independent'
stateNames = metadata.stateNames;
case '-dependent'
stateNames = metadata.dependentStateNames;
otherwise
throw(MException('DiskLogger:error', 'Invalid stateNames option ''%s'', stateNames'));
end
else
logN = DiskLogger.loadStructFromDisk(sprintf('%s/state-%d.mat', outputDirectory, numSegments), stateNames(:, 1));
metaStates.names = intersect(fieldnames(logN), unique(stateNames(:, 1)));
end
metaStates.properties = [stateNames cell(size(stateNames, 1), 2)];
for i = 1:size(stateNames, 1)
tmp = logN.(stateNames{i, 1}).(stateNames{i, 2});
metaStates.properties{i, 3} = class(tmp);
metaStates.properties{i, 4} = [size(tmp, 1) size(tmp, 2)];
if size(stateNames, 2) >= 3
if ischar(stateNames{i, 3}) && ismember(stateNames{i, 3}, {'-sum'; '-nnz'})
metaStates.properties{i, 3} = 'double';
metaStates.properties{i, 4}(1) = 1;
elseif isnumeric(stateNames{i, 3})
metaStates.properties{i, 4}(1) = size(stateNames{i, 3}, 1);
end
end
if size(stateNames, 2) >= 4
if ischar(stateNames{i, 4}) && ismember(stateNames{i, 4}, {'-sum'; '-nnz'})
metaStates.properties{i, 3} = 'double';
metaStates.properties{i, 4}(2) = 1;
elseif isnumeric(stateNames{i, 4})
metaStates.properties{i, 4}(2) = size(stateNames{i, 4}, 1);
end
end
end
states = DiskLogger.allocateMemory(metaStates, ceil((finTime-initTime) / downsampleStepSec));
%downsample timecourse
if strcmp(method, 'byState') && exist([outputDirectory filesep 'state-Time-values.mat'], 'file') && finTime - initTime > metadata.segmentSizeStep
switch downsampleType
case 'extract', states = DiskLogger.downSample_extract_byState(states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec);
case 'mean', states = DiskLogger.downSample_mean_byState( states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec);
end
else
switch downsampleType
case 'extract', states = DiskLogger.downSample_extract_byTime(states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec);
case 'mean', states = DiskLogger.downSample_mean_byTime( states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec);
end
end
end
function states = downSample_extract_byState(states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec)
import edu.stanford.covert.cell.sim.util.DiskLogger;
times = ceil(initTime/options.stepSizeSec : downsampleStepSec/options.stepSizeSec : ceil(finTime/downsampleStepSec)*downsampleStepSec/options.stepSizeSec) * options.stepSizeSec;
times(end) = finTime;
data = DiskLogger.loadStructFromDisk(sprintf('%s/state-%s-%s.mat', outputDirectory, 'Time', 'values'));
data = data.data;
allTimes = isequal(times, permute(data, [2 3 1]));
if allTimes
idxs = ':';
else
idxs = ismembc2(times, permute(data, [2 3 1]));
end
sNames = unique(stateNames(:, 1));
for i = 1:numel(sNames);
stateName = sNames{i};
propNames = stateNames(strcmp(stateNames(:, 1), stateName), 2:end);
for j = 1:size(propNames, 1)
fileName = sprintf('%s/state-%s-%s.mat', outputDirectory, stateName, propNames{j, 1});
if ~exist(fileName, 'file')
states = DiskLogger.downSample_extract_byTime(...
states, options, metadata, outputDirectory, [{stateName} propNames(j, :)], initTime, finTime, downsampleStepSec);
continue;
end
tmp = DiskLogger.loadStructFromDisk(fileName);
tmp = tmp.data;
calcMargins = false(1, 2);
calcSums = false(1, 2);
calcNnzs = false(1, 2);
s.type = '()';
s.subs = {':' ':' ':'};
if size(stateNames, 2) > 2
s.subs(1:2) = propNames(j, 2:3);
calcMargins = cellfun(@(x) isnumeric(x) && size(x, 2) > 1, propNames(j, 2:3));
calcSums = strcmp(propNames(j, 2:3), '-sum');
calcNnzs = strcmp(propNames(j, 2:3), '-nnz');
s.subs(calcSums | calcNnzs) = {':'};
end
s.subs{3} = idxs;
tmp = subsref(tmp, s);
if calcMargins(1)
siz = size(tmp);
siz = [size(s.subs{1}) siz(2:end)];
tmp = permute(sum(reshape(tmp, siz), 2), [1 3 4 2]);
elseif calcSums(1)
tmp = sum(tmp, 1);
elseif calcNnzs(1)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 1);
end
if calcMargins(2)
siz = size(tmp);
siz = [siz(1) size(s.subs{2}) siz(3:end)];
tmp = permute(sum(reshape(tmp, siz), 3), [1 2 4 3]);
elseif calcSums(2)
tmp = sum(tmp, 2);
elseif calcNnzs(2)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 2);
end
if isa(tmp, 'edu.stanford.covert.util.SparseMat') && ...
~isa(states.(stateName).(propNames{j, 1}), 'edu.stanford.covert.util.SparseMat')
tmp = full(tmp);
end
states.(stateName).(propNames{j, 1}) = tmp;
clear tmp;
end
end
end
function states = downSample_mean_byState(states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec)
import edu.stanford.covert.cell.sim.util.DiskLogger;
times = ceil(initTime/options.stepSizeSec : downsampleStepSec/options.stepSizeSec : ceil(finTime/downsampleStepSec)*downsampleStepSec/options.stepSizeSec) * options.stepSizeSec;
data = DiskLogger.loadStructFromDisk(sprintf('%s/state-%s-%s.mat', outputDirectory, 'Time', 'values'));
data = data.data;
idxs = ismembc2(times, permute(data, [2 3 1]));
sNames = unique(stateNames(:, 1));
for i = 1:numel(sNames);
stateName = sNames{i};
propNames = stateNames(strcmp(stateNames(:, 1), stateName), 2:end);
for j = 1:size(propNames, 1)
fileName = sprintf('%s/state-%s-%s.mat', outputDirectory, stateName, propNames{j, 1});
if ~exist(fileName, 'file')
states = DiskLogger.downSample_mean_byTime(...
states, options, metadata, outputDirectory, [{stateName} propNames(j, :)], initTime, finTime, downsampleStepSec);
continue;
end
data = DiskLogger.loadStructFromDisk(fileName);
data = data.data;
calcMargins = false(1, 2);
calcSums = false(1, 2);
calcNnzs = false(1, 2);
s.type = '()';
s.subs = {':' ':' ':'};
if size(stateNames, 2) > 2
s.subs(1:2) = propNames(j, 2:3);
calcMargins = cellfun(@(x) isnumeric(x) && size(x, 2) > 1, propNames(j, 2:3));
calcSums = strcmp(propNames(j, 2:3), '-sum');
calcNnzs = strcmp(propNames(j, 2:3), '-nnz');
s.subs(calcSums | calcNnzs) = {':'};
end
if iscell(states.(stateName).(propNames{j, 1}))
s.subs{3} = idxs;
states.(stateName).(propNames{j, 1}) = subsref(data, s);
else
for k = 1:numel(idxs)
s.subs{3} = min(finTime, idxs(k)+(0:downsampleStepSec-1));
tmp = mean(subsref(data, s), 3);
if calcMargins(1)
siz = size(tmp);
siz = [size(s.subs{1}) siz(2:end)];
tmp = permute(sum(reshape(tmp, siz), 2), [1 3 4 2]);
elseif calcSums(1)
tmp = sum(tmp, 1);
elseif calcNnzs(1)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 1);
end
if calcMargins(2)
siz = size(tmp);
siz = [siz(1) size(s.subs{2}) siz(3:end)];
tmp = permute(sum(reshape(tmp, siz), 3), [1 2 4 3]);
elseif calcSums(2)
tmp = sum(tmp, 2);
elseif calcNnzs(2)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 2);
end
states.(stateName).(propNames{j, 1})(:, :, k) = tmp;
clear tmp;
end
end
clear data;
end
end
end
function states = downSample_extract_byTime(states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec)
import edu.stanford.covert.cell.sim.util.DiskLogger;
numSegments = ceil(metadata.lengthSec / options.stepSizeSec / metadata.segmentSizeStep);
times = ceil(initTime/options.stepSizeSec : downsampleStepSec/options.stepSizeSec : ceil(finTime/downsampleStepSec)*downsampleStepSec/options.stepSizeSec) * options.stepSizeSec;
times(end) = finTime;
tmp2 = cell(numSegments, size(stateNames, 1));
for i = 1:numSegments
if ~any(ismembc(times, (i-1) * metadata.segmentSizeStep + (1:metadata.segmentSizeStep)))
continue;
end
try
logI = DiskLogger.loadStructFromDisk(sprintf('%s/state-%d.mat', outputDirectory, i), [stateNames(:, 1); 'Time']);
catch exception
exception.addCause(MException('DiskLogger:error', ...
'Unable to load segment %d of states {%s}', ...
i, strjoin(', ', stateNames{:, 1}))).rethrow();
end
allTimes = all(ismembc(logI.Time.values, times));
segIdxs = ismembc2(times, logI.Time.values);
gblIdxs = find(segIdxs > 0);
segIdxs = segIdxs(gblIdxs);
for j = 1:size(stateNames, 1)
try
calcMargins = false(1, 2);
calcSums = false(1, 2);
calcNnzs = false(1, 2);
s.type = '()';
s.subs = {':' ':' ':'};
if size(stateNames, 2) > 2
s.subs(1:2) = stateNames(j, 3:4);
calcMargins = cellfun(@(x) isnumeric(x) && size(x, 2) > 1, stateNames(j, 3:4));
calcSums = strcmp(stateNames(j, 3:4), '-sum');
calcNnzs = strcmp(stateNames(j, 3:4), '-nnz');
s.subs(calcSums | calcNnzs) = {':'};
end
if ~allTimes
s.subs{3} = segIdxs;
end
tmp = subsref(logI.(stateNames{j, 1}).(stateNames{j, 2}), s);
if calcMargins(1)
siz = size(tmp);
siz = [size(s.subs{1}) siz(2:end)];
tmp = permute(sum(reshape(tmp, siz), 2), [1 3 4 2]);
elseif calcSums(1)
tmp = sum(tmp, 1);
elseif calcNnzs(1)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 1);
end
if calcMargins(2)
siz = size(tmp);
siz = [siz(1) size(s.subs{2}) siz(3:end)];
tmp = permute(sum(reshape(tmp, siz), 3), [1 2 4 3]);
elseif calcSums(2)
tmp = sum(tmp, 2);
elseif calcNnzs(2)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 2);
end
if isa(states.(stateNames{j, 1}).(stateNames{j, 2}), 'edu.stanford.covert.util.SparseMat')
if ~isa(tmp, 'edu.stanford.covert.util.SparseMat')
tmp = edu.stanford.covert.util.SparseMat(tmp);
end
tmp2{i, j} = tmp;
else
states.(stateNames{j, 1}).(stateNames{j, 2})(1:size(tmp, 1), 1:size(tmp, 2), gblIdxs) = tmp;
end
catch exception
exception.addCause(MException('DiskLogger:error', ...
'Unable to load segment %d of state %s.%s', ...
i, stateNames{j, 1}, stateNames{j, 2})).rethrow();
end
clear tmp;
end
clear logI;
end
for j = 1:size(stateNames, 1)
if isa(states.(stateNames{j, 1}).(stateNames{j, 2}), 'edu.stanford.covert.util.SparseMat')
tfs = ~cellfun(@isempty, tmp2(:, j));
tmp = tmp2(tfs, j);
siz = [0 0];
for i = 1:numel(tmp)
siz(1) = max(siz(1), size(tmp{i}, 1));
siz(2) = max(siz(2), size(tmp{i}, 2));
end
for i = 1:numel(tmp)
if size(tmp{i}, 1) ~= siz(1)
tmp{i} = [tmp{i}; edu.stanford.covert.util.SparseMat([], [], [siz(1)-size(tmp{i}, 1) size(tmp{i}, 2) size(tmp{i}, 3)])];
end
if size(tmp{i}, 2) ~= siz(2)
tmp{i} = [tmp{i} edu.stanford.covert.util.SparseMat([], [], [size(tmp{i}, 1) siz(2)-size(tmp{i}, 2) size(tmp{i}, 3)])];
end
end
states.(stateNames{j, 1}).(stateNames{j, 2}) = cat(3, tmp{:});
clear tmp;
end
end
end
function states = downSample_mean_byTime(states, options, metadata, outputDirectory, stateNames, initTime, finTime, downsampleStepSec)
import edu.stanford.covert.cell.sim.util.DiskLogger;
numSegments = ceil(metadata.lengthSec / options.stepSizeSec / metadata.segmentSizeStep);
times = ceil(initTime/options.stepSizeSec : downsampleStepSec/options.stepSizeSec : ceil(finTime/downsampleStepSec)*downsampleStepSec/options.stepSizeSec) * options.stepSizeSec;
log2 = DiskLogger.loadStructFromDisk(sprintf('%s/state-%d.mat', outputDirectory, 1), [stateNames(:,1); 'Time']);
for i = 1:numSegments
log1 = log2;
if i < numSegments
try
log2 = DiskLogger.loadStructFromDisk(sprintf('%s/state-%d.mat', outputDirectory, i+1), [stateNames(:,1); 'Time']);
catch exception
exception.addCause(MException('DiskLogger:error', ...
'Unable to load segment %d of states {%s}', ...
i, strjoin(', ', stateNames{:, 1}))).rethrow();
end
end
segIdxs = ismembc2(times, log1.Time.values);
gblIdxs = find(segIdxs > 0);
segIdxs = segIdxs(gblIdxs);
for k = 1:numel(segIdxs)
for j = 1:size(stateNames, 1)
try
tmpSegIdxs = segIdxs(k)+(1:downsampleStepSec/options.stepSizeSec)-1;
segmentLength = size(log1.(stateNames{j, 1}).(stateNames{j, 2}), 3);
calcMargins = false(1, 2);
calcSums = false(1, 2);
calcNnzs = false(1, 2);
s.type = '()';
s.subs = {':' ':' ':'};
if size(stateNames, 2) > 2
s.subs(1:2) = stateNames(j, 3:4);
calcMargins = cellfun(@(x) isnumeric(x) && size(x, 2) > 1, stateNames(j, 3:4));
calcSums = strcmp(stateNames(j, 3:4), '-sum');
calcNnzs = strcmp(stateNames(j, 3:4), '-nnz');
s.subs(calcSums | calcNnzs) = {':'};
end
if tmpSegIdxs(end) <= segmentLength
s.subs{3} = tmpSegIdxs;
tmp = subsref(log1.(stateNames{j, 1}).(stateNames{j, 2}), s);
else
s1 = s;
s2 = s;
s1.subs{3} = tmpSegIdxs(1):size(log1.(stateNames{j, 1}).(stateNames{j, 2}), 3);
s2.subs{3} = 1:tmpSegIdxs(end)-segmentLength;
tmp = cat(3, ...
subsref(log1.(stateNames{j, 1}).(stateNames{j, 2}), s1), ...
subsref(log2.(stateNames{j, 1}).(stateNames{j, 2}), s2));
end
if calcMargins(1)
siz = size(tmp);
siz = [size(s.subs{1}) siz(2:end)];
tmp = permute(sum(reshape(tmp, siz), 2), [1 3 4 2]);
elseif calcSums(1)
tmp = sum(tmp, 1);
elseif calcNnzs(1)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 1);
end
if calcMargins(2)
siz = size(tmp);
siz = [siz(1) size(s.subs{2}) siz(3:end)];
tmp = permute(sum(reshape(tmp, siz), 3), [1 2 4 3]);
elseif calcSums(2)
tmp = sum(tmp, 2);
elseif calcNnzs(2)
if iscell(tmp)
tmp = ~cellfun(@isempty, tmp);
end
tmp = sum(tmp ~= 0, 2);
end
switch class(tmp)
case 'cell', tmp = tmp(:, :, 1);
otherwise, tmp = mean(tmp, 3);
end
if isa(states.(stateNames{j, 1}).(stateNames{j, 2}), 'edu.stanford.covert.util.SparseMat')
if gblIdxs(k) == 1
if ~isa(tmp, 'edu.stanford.covert.util.SparseMat')
tmp = edu.stanford.covert.util.SparseMat(tmp);
end
states.(stateNames{j, 1}).(stateNames{j, 2}) = tmp;
else
states.(stateNames{j, 1}).(stateNames{j, 2}) = cat(3, ...
states.(stateNames{j, 1}).(stateNames{j, 2}), ...
tmp);
end
else
states.(stateNames{j, 1}).(stateNames{j, 2})(1:size(tmp,1), 1:size(tmp,2), gblIdxs(k)) = tmp;
end
catch exception
exception.addCause(MException('DiskLogger:error', ...
'Unable to load segment %d %s.%s', ...
i, stateNames{j, 1}, stateNames{j, 2})).rethrow();
end
clear tmp;
end
end
clear log1;
end
clear log2;
end
function saveStructToDisk(outputDirectory, path, data, structTF, appendTf)
options = {'-v7'};
if nargin >= 5 && appendTf
options = [options; '-append'];
end
if (nargin < 4 && isstruct(data)) || (nargin >= 4 && structTF)
options = [options; '-struct'];
end
lastwarn('');
save([outputDirectory filesep path], options{:}, 'data');
[~, id] = lastwarn();
if isequal(id, 'MATLAB:save:sizeTooBigForMATFile')
tmp = whos('data');
if tmp.bytes < 5e9
options{1} = '-v7.3';
save([outputDirectory filesep path], options{:}, 'data');
else
warning('WholeCell:warning', 'Data too large to store as mat file ''%s''', [outputDirectory filesep path]);
end
end
end
function structure = loadStructFromDisk(path, fields, varargin)
if nargin == 1
structure = load(path);
elseif nargin == 2 && iscell(fields)
fields = unique(fields);
structure = load(path, fields{:});
elseif nargin == 2
structure = load(path, fields);
elseif nargin > 2
structure = load(path, fields, varargin{:});
end
end
%Allocates space for one segment's worth of time course data.
function log = allocateMemory(metaStates, segmentSizeStep)
import edu.stanford.covert.cell.sim.util.DiskLogger;
log = struct;
for i = 1:size(metaStates.names, 1)
log.(metaStates.names{i, 1}) = struct;
end
for i = 1:size(metaStates.properties, 1)
log.(metaStates.properties{i, 1}).(metaStates.properties{i, 2}) = DiskLogger.allocateData(...
metaStates.properties{i, 3}, [metaStates.properties{i, 4}, segmentSizeStep]);
end
end
function data = allocateData(dataType, sz, sparsity)
import edu.stanford.covert.cell.sim.util.DiskLogger;
import edu.stanford.covert.util.CircularSparseMat;
import edu.stanford.covert.util.SparseMat;
if prod(sz) > 10^8 && (nargin < 3 || sparsity < 1/3) && ...
ismember(dataType, {'logical', 'double', 'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16' ,'int32', 'int64'})
dataType = 'edu.stanford.covert.util.SparseMat';
end
switch dataType
case 'edu.stanford.covert.util.SparseMat', data = SparseMat([], [], sz);
case 'edu.stanford.covert.util.CircularSparseMat', data = CircularSparseMat([], [], sz, 1);
case 'char', data = char(zeros(sz));
case 'logical', data = false(sz);
case 'cell', data = cell(sz);
otherwise,
try
data = zeros(sz, dataType);
catch exception
exception.addCause(MException('DiskLogger:invalidDataType', '%s is not a numeric data type', dataType)).rethrow();
end
end
end
end
methods (Static = true)
function reindexTimeCourses(outputDirectory)
import edu.stanford.covert.cell.sim.util.DiskLogger;
%% get meta data, options
metadata = DiskLogger.loadMetadata(outputDirectory);
options = DiskLogger.loadOptions(outputDirectory);
stateNames = DiskLogger.getAvailableStates(outputDirectory);
%% re-index simulation, organized by state
for i = 1:size(stateNames, 1)
try
data = DiskLogger.loadTimecourses(outputDirectory, ...
stateNames(i, :), ...
1, metadata.lengthSec, metadata.downsampleStepSec, 'extract', ...
options, metadata, 'byTime');
DiskLogger.saveStructToDisk(outputDirectory, ['state-' stateNames{i, 1} '-' stateNames{i, 2} '.mat'], ...
data.(stateNames{i, 1}).(stateNames{i, 2}), false);
catch exception
if ismember(exception.identifier, {'MATLAB:save:errorClosingFile'; 'MATLAB:nomem'})
warning('WholeCell:warning:reindexing', 'Unable to reindex %s.%s\n%s', stateNames{i, 1}, stateNames{i, 2}, exception.getReport());
else
exception.addCause(MException('DiskLogger:error', 'Unable to reindex %s.%s', stateNames{i, 1}, stateNames{i, 2})).rethrow();
end
end
%clean up
clear data;
end
end
end
end