psom_test_io.m

function [pipel,opt_pipe] = psom_test_io(path_test,opt)
% A test pipeline with chains of jobs reading/writing on disk
%
% [pipe,opt_pipe] = psom_test_io(path_test,opt)
%
% PATH_TEST (string, default current path) where to run the test.
% OPT (structure) any option passed to PSOM will do. In addition the 
%   following options are available:
%   PATH_LOGS is forced to [path_test filesep 'logs']
%   SIZE (float, default 10) the size of data generated by each job, in megabytes.
%      Note that the closest size possible with an array in double precision
%      is selected. 
%   TIME (scalar, default 1) each job will sleep for TIME+TIME_RAND before writing to disk. 
%   TIME_RAND (scalar, default 0) See TIME. Each job will get a random value assigned, 
%      distributed uniformly between 0 and TIME_RAND.
%   NB_JOBS (integer, default 1) the number of jobs.
%   NB_CHAINS (integer, default 100) the number of chains. The final 
%     number of jobs is NB_JOBS x NB_CHAINS.
%   FLAG_READ (boolean, default true) if flag_read is true, each job reads 
%     the output of the preceding job in the chain (except of course the 
%     first one). Otherwise it only writes.
%   FLAG_TEST (boolean, default false) if FLAG_TEST is on, the pipeline
%     is generated but not executed.
% PIPE (structure) the pipeline.
% OPT_PIPE (structure) the options to run the pipeline.
%
% Note: 1 MB is defined here as 10^3 kB, or 10^6 bytes, consistent with the 
% international system of units. This is slightly different from the definition 
% used by linux and windows file systems where 1 MB = 1024 kB
%
% Copyright (c) Pierre Bellec, 
% Departement d'informatique et de recherche operationnelle
% Centre de recherche de l'institut de Geriatrie de Montreal
% Universite de Montreal, 2015.
% Maintainer : pierre.bellec@criugm.qc.ca
% See licensing information the LICENSE file.
% Keywords : pipeline, PSOM, test

%% Set up default options
pipel = struct;

if nargin < 2
    opt = struct;
end

list_opt = { 'time' , 'time_rand' , 'flag_read' , 'size' , 'nb_jobs' , 'nb_chains' , 'flag_test' };
list_def = { 3      , 0           , true        , 10     , 1         , 100         , false       };
opt = psom_struct_defaults(opt,list_opt,list_def,false);

if (nargin < 1)||isempty(path_test)
    path_test = pwd;
end
if ~strcmp(path_test(end),filesep)
    path_test = [path_test filesep];
end

opt.path_logs = [path_test 'logs'];

%% The options for PSOM
opt_pipe = rmfield(opt,list_opt);

%% compute the size of the array
nb_vec = round(10^6*(opt.size/8));

%% Build the pipeline
for num_j = 1:opt.nb_jobs
    for num_c = 1:opt.nb_chains
        job_name = sprintf('job%i_%i',num_j,num_c);   
        pipel.(job_name).opt.nb_vec = nb_vec;
        inst_read = [ ...
                    'hf = fopen(files_in,''r'');' ...
                    ' data0 = fread(hf,Inf,''double'');' ...
                    ' fclose(hf);' ...
                    ];
        inst_write = [ ...                          
                     sprintf('system(''sleep %1.3f'');',opt.time+opt.time_rand*rand(1)) ...
                     ' data = rand(opt.nb_vec,1);' ...
                     ' hf = fopen(files_out,''w'');' ...
                     ' fwrite(hf,data,''double'');' ...
                     ' fclose(hf);' ...
                     ];
        if (num_j > 1)&&opt.flag_read
            pipel.(job_name).command = [inst_read inst_write];
            pipel.(job_name).files_in = sprintf('%sdata_job%i_%i.dat',path_test,num_j-1,num_c);
        else
            pipel.(job_name).command = inst_write;       
        end
        pipel.(job_name).files_out = sprintf('%sdata_job%i_%i.dat',path_test,num_j,num_c);   
    end
end

%% Run the pipeline
if ~opt.flag_test
    psom_run_pipeline(pipel,opt_pipe);
end