Merge pull request automaticanalysis#149 from tiborauer/master

Success! Finally! Merging!

aaClass.m

@@ -22,6 +22,7 @@
                 fprintf('\nPlease wait a moment, adding <a href = "matlab: cd %s">%s</a> to the path\n',obj.Path,obj.Name);
                 addpath(genpath(obj.Path)); % recursively add AA subfolders
                 rmpath(genpath(fullfile(obj.Path,'.git'))); % remove GitHub-related path
+                rmpath(genpath(fullfile(obj.Path,'external','cluster'))); % remove cluster-integration path
             end
 
             obj.Name = 'automaticanalysis';            

aa_engine/aa_doprocessing.m

@@ -74,7 +74,7 @@
 
 
 function [aap]=aa_doprocessing(aap,username,bucket,bucketfordicom,workerid,analysisid,jobid)
-%#function aaq_condor aaq_localsingle aaq_matlab_pct aaq_qsub
+%#function aaq_condor aaq_localsingle aaq_matlab_pct aaq_qsub aaq_qsub_debug aaq_qsub_monitor_jobs aaq_qsub_nonDCS
 
 aap = aa_init(aap);
 
@@ -184,52 +184,22 @@
 % Run initialisation modules
 aap=aas_doprocessing_initialisationmodules(aap);
 
-% THE MODULES IN AAP.TASKLIST.STAGES ARE RUN IF A CORRESPONDING DONE_ FLAG
-% IS NOT FOUND. ONE IS CREATED AFTER SUCCESSFUL EXECUTION
-% Now run stage-by-stage tasks
-
 % get dependencies of stages, referenced in both directions (aap.internal.dependenton
 % and aap.internal.dependencyof)
-
 aap=aas_builddependencymap(aap);
 
-% Create folder (required by aas_findinputstreamsources to save provenance)
-if (strcmp(aap.directory_conventions.remotefilesystem,'none'))
-    aapsavepth=fullfile(aap.acq_details.root,[aap.directory_conventions.analysisid aap.directory_conventions.analysisid_suffix]);
-    if (isempty(dir(aapsavepth)))
-        [s w]=aas_shell(['mkdir ' aapsavepth]);
-        if (s)
-            aas_log(aap,1,sprintf('Problem making directory%s',aapsavepth));
-        end
-    end
-end
-
-% Use input and output stream information in XML header to find
-% out what data comes from where and goes where
-aap=aas_findinputstreamsources(aap);
-
-% Store these initial settings before any module specific customisation
-aap.internal.aap_initial=aap;
-aap.internal.aap_initial.aap.internal.aap_initial=[]; % Prevent recursively expanding storage
-
-% Save AAP structure
-%  could save aaps somewhere on S3 too?
-studypath=aas_getstudypath(aap);
+% - find out what data comes from where and goes where
+% - store initial settings before any module specific customisation
+% - save AAP structure
+aap = update_aap(aap);
 
 % check disk space
-FileObj=java.io.File(studypath);
+FileObj=java.io.File(aas_getstudypath(aap));
 GBFree=FileObj.getUsableSpace/1024/1000/1000;
 if (GBFree<10)
     aas_log(aap,false,sprintf('WARNING: Only %f GB of disk space free on analysis drive',GBFree));
 end;
 
-if (strcmp(aap.directory_conventions.remotefilesystem,'none'))
-    aapsavefn=fullfile(aapsavepth,'aap_parameters');
-    aap.internal.aapversion=aa.Version;
-    aap.internal.aappath=aa.Path;
-    save(aapsavefn,'aap');
-end
-
 % Choose where to run all tasks
 if ~exist(sprintf('aaq_%s', aap.options.wheretoprocess),'file'), 
     aas_log(aap,true,sprintf('Unknown aap.options.wheretoprocess: %s\n',aap.options.wheretoprocess));
@@ -258,10 +228,9 @@
 end
 
 %% Main task loop
-mytasks={'checkrequirements','doit'}; %
-for l=1:length(mytasks)
+for mytasks = {'checkrequirements','doit'} %
     for k=1:length(aap.tasklist.main.module)
-        task=mytasks{l};
+        task=mytasks{1};
         % allow full path of module to be provided [djm]
         [stagepath stagename]=fileparts(aap.tasklist.main.module(k).name);
         index=aap.tasklist.main.module(k).index;
@@ -380,17 +349,24 @@
 
                 aas_log(aap,false,msg);
             end
+            % Get jobs started as quickly as possible - important on AWS as it
+            % can take a while to scan all of the done flags
+            taskqueue.runall(dontcloseexistingworkers, false);
+            if ~isempty(localtaskqueue.jobqueue),
+                localtaskqueue.runall(dontcloseexistingworkers, false);
+            end
+            if ~taskqueue.isOpen, break; end
         end
-        % Get jobs started as quickly as possible - important on AWS as it
-        % can take a while to scan all of the done flags
-        taskqueue.runall(dontcloseexistingworkers, false);
-        if ~isempty(localtaskqueue.jobqueue), 
-            localtaskqueue.runall(dontcloseexistingworkers, false); 
-        end
-        if ~taskqueue.isOpen, break; end
     end
-    % Wait until all the jobs have finished
-    taskqueue.runall(dontcloseexistingworkers, true);
+    switch task
+        case 'checkrequirements'
+            % update map
+            aap = update_aap(aap);
+            taskqueue.aap = aap;
+        case 'doit'
+            % Wait until all the jobs have finished
+            taskqueue.runall(dontcloseexistingworkers, true);
+    end
 end
 
 if taskqueue.isOpen, taskqueue.close; end
@@ -422,6 +398,45 @@
 
 end
 
+function aap = update_aap(aap)
+global aa
+
+% restore root
+if isfield(aap.tasklist,'currenttask')
+    aap.acq_details.root = aap.internal.aap_initial.acq_details.root; 
+    aap.directory_conventions.analysisid = aap.internal.aap_initial.directory_conventions.analysisid;
+    aap.directory_conventions.analysisid_suffix = aap.internal.aap_initial.directory_conventions.analysisid_suffix;
+end
+
+% Create folder (required by aas_findinputstreamsources to save provenance)
+if (strcmp(aap.directory_conventions.remotefilesystem,'none'))
+    aapsavepth=fullfile(aap.acq_details.root,[aap.directory_conventions.analysisid aap.directory_conventions.analysisid_suffix]);
+    aas_makedir(aap,aapsavepth);
+end
+
+% Use input and output stream information in XML header to find
+% out what data comes from where and goes where
+aap=aas_findinputstreamsources(aap);
+
+if ~isfield(aap.tasklist,'currenttask') % Store these initial settings before any module specific customisation
+    aap.internal.aap_initial=aap;
+    aap.internal.aap_initial.aap.internal.aap_initial=[]; % Prevent recursively expanding storage
+else % Restore initial settings
+    initinternal = aap.internal;
+    aap = aap.internal.aap_initial;
+    aap.internal = initinternal;
+    if isfield(aap,'aap'), aap = rmfield(aap,'aap'); end
+end
+
+% Save AAP structure (S3?)
+if (strcmp(aap.directory_conventions.remotefilesystem,'none'))
+    aapsavefn=fullfile(aapsavepth,'aap_parameters');
+    aap.internal.aapversion=aa.Version;
+    aap.internal.aappath=aa.Path;
+    save(aapsavefn,'aap');
+end
+end
+
 
 function [loopvar]=getparallelparts(aap,stagenum)
 [prev_stagepath prev_stagename]=fileparts(aap.tasklist.main.module(stagenum).name);

aa_engine/aa_doprocessing_onetask.m

@@ -163,7 +163,7 @@
 
                     [gotinputs, streamfiles{inpind}]=aas_retrieve_inputs_part1(aap,inp,allinputs,deps);
                     if isempty(setdiff(gotinputs,allinputs)) && inp.isessential % no new inputs found
-                        aas_log(aap,true,sprintf('No inputs obtained for stream %s',inp.name));
+                        aas_log(aap,true,sprintf('No inputs obtained for stream %s!\n\tModule %s might not have crteated it.',inp.name,inp.sourcestagename));
                     end;
                     allinputs=[allinputs;gotinputs];
                 end;

aa_engine/aa_feval.m

@@ -1,6 +1,6 @@
 % This function is automagically generated by aa_build_standalone_1_pragmas
 function varargout=aa_feval(funcname,varargin)
-%#function aamod_ANTS_EPIwarp aamod_ANTS_build_MMtemplate aamod_ANTS_build_template aamod_ANTS_struc2epi aamod_ANTS_struc2template aamod_ANTS_warp aamod_GLMdenoise aamod_LoAd aamod_LoAd2SPM aamod_LouvainCluster aamod_MP2RAGE aamod_MTI2MTR aamod_autoidentifyseries aamod_autoidentifyseries_siemens aamod_bet aamod_bet_epi_masking aamod_bet_epi_reslicing aamod_bet_freesurfer aamod_bet_premask aamod_biascorrect aamod_biascorrect_ANTS aamod_biascorrect_segment8 aamod_biascorrect_segment8_multichan aamod_binarizeimage aamod_binaryfromlabels aamod_brainmask aamod_brainmaskcombine aamod_checkparameters aamod_clusteringXValidation aamod_compSignal aamod_compareoverlap aamod_convert_diffusion aamod_convert_dmdx aamod_convert_epis aamod_convert_fieldmaps aamod_convert_specialseries aamod_convert_structural aamod_copy_image_orientation aamod_coreg_extended aamod_coreg_extended_1 aamod_coreg_extended_2 aamod_coreg_general aamod_coreg_noss aamod_coreg_structural2fa aamod_coreg_structural2template aamod_coregisterstructural2template aamod_dartel_createtemplate aamod_dartel_denorm aamod_dartel_normmni aamod_decodeDMLT aamod_denoiseANLM aamod_diffusion_bedpostx aamod_diffusion_dki_tractography_prepare aamod_diffusion_dkifit aamod_diffusion_dtifit aamod_diffusion_dtinlfit aamod_diffusion_eddy aamod_diffusion_eddycorrect aamod_diffusion_extractnodif aamod_diffusion_probtrackx aamod_diffusion_probtrackxsummarize_group aamod_diffusion_probtrackxsummarize_indv aamod_diffusion_roi_valid aamod_diffusion_topup aamod_diffusionfromnifti aamod_epifromnifti aamod_evaluatesubjectnames aamod_facemasking aamod_fconn_computematrix aamod_fconnmatrix_seedseed aamod_fieldmap2VDM aamod_fieldmapfromnifti aamod_firstlevel_contrasts aamod_firstlevel_model aamod_firstlevel_model_MVPaa aamod_firstlevel_modelspecify aamod_firstlevel_threshold aamod_freesurfer_autorecon aamod_freesurfer_deface aamod_freesurfer_deface_apply aamod_freesurfer_initialise aamod_freesurfer_register aamod_fsl_FAST aamod_fsl_FIRST aamod_fsl_reorienttoMNI aamod_fsl_robustFOV aamod_fslmaths aamod_fslmerge aamod_fslsplit aamod_garbagecollection aamod_get_dicom_ASL aamod_get_dicom_diffusion aamod_get_dicom_epi aamod_get_dicom_fieldmap aamod_get_dicom_specialseries aamod_get_dicom_structural aamod_get_tSNR aamod_ggmfit aamod_highpass aamod_highpassfilter_epi aamod_imcalc aamod_importfilesasstream aamod_input_staging aamod_listspikes aamod_make_epis_float aamod_mapstreams aamod_marsbar aamod_mask_fromsegment aamod_mask_fromstruct aamod_maths aamod_meanepitimecourse aamod_meg_average aamod_meg_convert aamod_meg_denoise_ICA_1 aamod_meg_denoise_ICA_2_applytrajectory aamod_meg_epochs aamod_meg_get_fif aamod_meg_grandmean aamod_meg_maxfilt aamod_meg_merge aamod_melodic aamod_mirrorandsubtract aamod_modelestimate aamod_movie aamod_moviecorr_meantimecourse aamod_moviecorr_summary aamod_newsubj_init aamod_norm_noss aamod_norm_vbm aamod_norm_write aamod_norm_write_dartel aamod_normalisebytotalgrey aamod_oneway_ANOVA aamod_pewarp_estimate aamod_pewarp_write aamod_possum aamod_ppi_model aamod_ppi_prepare aamod_realign aamod_realignunwarp aamod_reorientto aamod_reorienttomiddle aamod_reslice aamod_resliceROI aamod_reslice_rois aamod_roi_extract aamod_roi_valid aamod_rois_getvalues aamod_secondlevel_GIFT aamod_secondlevel_contrasts aamod_secondlevel_model aamod_secondlevel_randomise aamod_secondlevel_threshold aamod_seedConnectivity aamod_segment aamod_segment8 aamod_segmentvbm8 aamod_slicetiming aamod_smooth aamod_smooth_structurals aamod_split aamod_structural_overlay aamod_structuralfromnifti aamod_structuralstats aamod_study_init aamod_tSNR_EPI aamod_temporalfilter aamod_tensor_ica aamod_tissue_spectrum aamod_tissue_spectrum_summarize aamod_tissue_wavelets aamod_tissue_wavelets_summarize aamod_trimEPIVols aamod_tsdiffana aamod_unnormalise_rois aamod_unnormalise_rois2 aamod_unzipstream aamod_vois_extract aamod_waveletdespike aamod_MVPaa_brain_1st aamod_MVPaa_brain_SPM aamod_MVPaa_roi_1st aamod_MVPaa_roi_2nd aamod_template_session
+%#function aamod_ANTS_EPIwarp aamod_ANTS_build_MMtemplate aamod_ANTS_build_template aamod_ANTS_struc2epi aamod_ANTS_struc2template aamod_ANTS_warp aamod_ASL aamod_GLMdenoise aamod_LI aamod_LoAd aamod_LoAd2SPM aamod_LouvainCluster aamod_MP2RAGE aamod_MPM aamod_MTI2MTR aamod_applyVDM aamod_autoidentifyseries aamod_bet aamod_bet_epi_masking aamod_bet_epi_reslicing aamod_bet_freesurfer aamod_bet_premask aamod_biascorrect aamod_biascorrect_ANTS aamod_biascorrect_segment8 aamod_biascorrect_segment8_multichan aamod_binarizeimage aamod_binaryfromlabels aamod_brainmask aamod_brainmaskcombine aamod_checkparameters aamod_clusteringXValidation aamod_compSignal aamod_compareoverlap aamod_convert_diffusion aamod_convert_dmdx aamod_convert_epis aamod_convert_fieldmaps aamod_convert_specialseries aamod_convert_structural aamod_copy_image_orientation aamod_coreg_extended aamod_coreg_extended_1 aamod_coreg_extended_2 aamod_coreg_general aamod_coreg_noss aamod_coreg_structural2fa aamod_coreg_structural2template aamod_coregisterstructural2template aamod_dartel_createtemplate aamod_dartel_denorm aamod_dartel_normmni aamod_decodeDMLT aamod_denoiseANLM aamod_diffusion_applytopup aamod_diffusion_bedpostx aamod_diffusion_dki_tractography_prepare aamod_diffusion_dkifit aamod_diffusion_dtifit aamod_diffusion_dtinlfit aamod_diffusion_eddy aamod_diffusion_eddycorrect aamod_diffusion_extractnodif aamod_diffusion_probtrackx aamod_diffusion_probtrackxsummarize_group aamod_diffusion_probtrackxsummarize_indv aamod_diffusion_topup aamod_diffusionfromnifti aamod_epifromnifti aamod_evaluatesubjectnames aamod_facemasking aamod_fconn_computematrix aamod_fconnmatrix_seedseed aamod_fieldmap2VDM aamod_fieldmapfromnifti aamod_firstlevel_contrasts aamod_firstlevel_model aamod_firstlevel_model_1_config aamod_firstlevel_model_2_convolve aamod_firstlevel_model_3_estimate aamod_firstlevel_model_MVPaa aamod_firstlevel_model_collapsepredictors aamod_firstlevel_model_subrunsplit aamod_firstlevel_threshold aamod_freesurfer_autorecon aamod_freesurfer_deface aamod_freesurfer_deface_apply aamod_freesurfer_initialise aamod_freesurfer_register aamod_fsl_FAST aamod_fsl_FIRST aamod_fsl_reorienttoMNI aamod_fsl_robustFOV aamod_fslmaths aamod_fslmerge aamod_fslsplit aamod_garbagecollection aamod_get_dicom_diffusion aamod_get_dicom_epi aamod_get_dicom_fieldmap aamod_get_dicom_specialseries aamod_get_dicom_structural aamod_get_tSNR aamod_ggmfit aamod_highpass aamod_highpassfilter_epi aamod_importfilesasstream aamod_input_staging aamod_listspikes aamod_make_epis_float aamod_mapstreams aamod_marsbar aamod_mask_fromsegment aamod_mask_fromstruct aamod_maths aamod_meanepitimecourse aamod_meanstructural aamod_meg_average aamod_meg_convert aamod_meg_denoise_ICA_1 aamod_meg_denoise_ICA_2_applytrajectory aamod_meg_epochs aamod_meg_get_fif aamod_meg_grandmean aamod_meg_maxfilt aamod_meg_merge aamod_melodic aamod_modelestimate aamod_movie aamod_moviecorr_meantimecourse aamod_moviecorr_summary aamod_newsubj_init aamod_norm_noss aamod_norm_vbm aamod_norm_write aamod_norm_write_dartel aamod_normalisebytotalgrey aamod_oneway_ANOVA aamod_pewarp_estimate aamod_pewarp_write aamod_possum aamod_ppi_prepare aamod_realign aamod_realignunwarp aamod_reorientto aamod_reorienttomiddle aamod_reslice aamod_resliceROI aamod_reslice_rois aamod_roi_extract aamod_roi_valid aamod_rois_getvalues aamod_secondlevel_GIFT aamod_secondlevel_contrasts aamod_secondlevel_model aamod_secondlevel_randomise aamod_secondlevel_threshold aamod_seedConnectivity aamod_segment aamod_segment8 aamod_segmentvbm8 aamod_slicetiming aamod_smooth aamod_smooth_structurals aamod_split aamod_structural_overlay aamod_structuralfromnifti aamod_structuralstats aamod_study_init aamod_tSNR_EPI aamod_temporalfilter aamod_tensor_ica aamod_tissue_spectrum aamod_tissue_spectrum_summarize aamod_tissue_wavelets aamod_tissue_wavelets_summarize aamod_trimEPIVols aamod_tsdiffana aamod_unnormalise_rois aamod_unnormalise_rois2 aamod_unzipstream aamod_vois_extract aamod_waveletdespike aamod_MVPaa_brain_1st aamod_MVPaa_brain_SPM aamod_MVPaa_roi_1st aamod_MVPaa_roi_2nd aamod_template_session
     try
         nout = max(nargout(funcname),1);
     catch myerr

aa_engine/aa_init.m

@@ -91,6 +91,14 @@
     fullfile(spm('Dir'),'toolbox', 'Neural_Models'),...
     fullfile(spm('Dir'),'toolbox', 'MEEGtools'));
 
+% Path fore spmtools
+if isfield(aap.directory_conventions,'spmtoolsdir') && ~isempty(aap.directory_conventions.spmtoolsdir)
+    SPMTools = textscan(aap.directory_conventions.spmtoolsdir,'%s','delimiter', ':'); SPMTools = SPMTools{1};
+    for pp = SPMTools'
+        addpath(genpath(pp{1}));
+    end
+end
+
 % Path for EEGLAB, if specified
 if ~isempty(aap.directory_conventions.eeglabdir)
     addpath(...
@@ -147,16 +155,6 @@
     end;
 end;
 
-% Path to VBQ
-if isfield(aap.directory_conventions,'VBQdir') && ~isempty(aap.directory_conventions.VBQdir)
-    addpath(aap.directory_conventions.VBQdir);
-else
-    % Check whether already in path, give warning if not
-    if isempty(which('vbq_mpr_b0_b1'))
-       aas_log(aap,false,sprintf('VBQ toolbox not found, if you need this you should add it to the matlab path manually, or set aap.directory_conventions.VBQdir'));
-    end;
-end;
-
 % Path to LI toolbox
 if isfield(aap.directory_conventions,'LIdir') && ~isempty(aap.directory_conventions.LIdir)
     addpath(aap.directory_conventions.LIdir);
@@ -191,10 +189,12 @@
 end
 % spmtools
 if isfield(aap.directory_conventions,'spmtoolsdir') && ~isempty(aap.directory_conventions.spmtoolsdir)
-    SPMTools = textscan(aap.directory_conventions.spmtoolsdir,'%s','delimiter', ':');
-    SPMTools = SPMTools{1};
+    SPMTools = textscan(aap.directory_conventions.spmtoolsdir,'%s','delimiter', ':'); SPMTools = SPMTools{1};
     for pp = SPMTools'
-        if exist(pp{1},'dir'), reqpath{end+1}=pp{1};end
+        if exist(pp{1},'dir')
+            pdir = textscan(genpath(pp{1}),'%s','delimiter', ':'); pdir = pdir{1};
+            reqpath = [reqpath; pdir];
+        end
     end
 end
 
@@ -238,14 +238,6 @@
     end
 end
 
-% VBQ
-if isfield(aap.directory_conventions,'VBQdir') && ~isempty(aap.directory_conventions.VBQdir)
-    p_ind = cell_index(p,aap.directory_conventions.VBQdir);
-    for ip = p_ind
-        reqpath{end+1} = p{ip};
-    end
-end
-
 % LI
 if isfield(aap.directory_conventions,'LIdir') && ~isempty(aap.directory_conventions.LIdir)
     p_ind = cell_index(p,aap.directory_conventions.LIdir);

aa_engine/aaq/QueueViewerClass.m

@@ -53,9 +53,9 @@
             obj.Open;
         end
 
-        function delete(obj)
-            obj.Pool = [];
-        end
+%         function delete(obj)
+%             obj.Pool = [];
+%         end
 
         function Open(obj)
             FontSize = [1.7*get(0,'DefaultUicontrolFontSize') get(0,'DefaultUicontrolFontSize')*0.8]; % H W
@@ -205,7 +205,7 @@ function setAutoUpdate(obj,val)
     %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% UTILS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     methods (Hidden=true)
         function str = TaskInfo(obj,Task)
-            elapsedtime = obj.DisplayHelper.getRunningDuration(Task.StartTime,Task.FinishTime);
+            elapsedtime = aas_getTaskDuration(Task);
             str = sprintf(['- Has been running on %s\n'...
                 '- For %s'],...
                 Task.Worker.Host,...