From 63bd5dd3374a9bfb398a51581eced4aa16cb4bc9 Mon Sep 17 00:00:00 2001 From: Nicholas Stroustrup Date: Thu, 11 Jan 2018 10:39:49 +0100 Subject: [PATCH] bugfix --- .../ns_machine_analysis_data_loader.cpp | 298 ++++++++++++++++++ .../ns_machine_analysis_data_loader.h | 95 ++++++ 2 files changed, 393 insertions(+) create mode 100644 ns_image_server/movement_analysis/ns_machine_analysis_data_loader.cpp create mode 100644 ns_image_server/movement_analysis/ns_machine_analysis_data_loader.h diff --git a/ns_image_server/movement_analysis/ns_machine_analysis_data_loader.cpp b/ns_image_server/movement_analysis/ns_machine_analysis_data_loader.cpp new file mode 100644 index 000000000..a66d945e9 --- /dev/null +++ b/ns_image_server/movement_analysis/ns_machine_analysis_data_loader.cpp @@ -0,0 +1,298 @@ +#include "ns_machine_analysis_data_loader.h" +#include "ns_image_server.h" +#include "ns_survival_curve.h" +#include "ns_time_path_image_analyzer.h" +#include +#include "ns_hand_annotation_loader.h" +#include "ns_hidden_markov_model_posture_analyzer.h" +bool ns_machine_analysis_region_data::load_from_db(const ns_death_time_annotation_set::ns_annotation_type_to_load & annotation_type_to_load,const ns_loading_details & details,const ns_64_bit region_id,ns_sql & sql){ + death_time_annotation_set.clear(); + metadata.region_id = region_id; + if (annotation_type_to_load == ns_death_time_annotation_set::ns_recalculate_from_movement_quantification_data) + return recalculate_from_saved_movement_quantification(region_id,sql); + + ns_image_server_results_subject results_subject; + results_subject.region_id = region_id; + std::vector files_to_open; + switch(annotation_type_to_load){ + case ns_death_time_annotation_set::ns_all_annotations: + case ns_death_time_annotation_set::ns_censoring_and_movement_states: + files_to_open.push_back(ns_image_server_results_storage::ns_worm_position_annotations); + files_to_open.push_back(ns_image_server_results_storage::ns_censoring_and_movement_transitions); + break; + case ns_death_time_annotation_set::ns_censoring_data: + case ns_death_time_annotation_set::ns_movement_transitions: + case ns_death_time_annotation_set::ns_censoring_and_movement_transitions: + files_to_open.push_back(ns_image_server_results_storage::ns_censoring_and_movement_transitions); + break; + case ns_death_time_annotation_set::ns_movement_states: + files_to_open.push_back(ns_image_server_results_storage::ns_worm_position_annotations); + case ns_death_time_annotation_set::ns_no_annotations: + break; + default: throw ns_ex("ns_machine_analysis_data_loader::Unknown annotation type request"); + } + bool could_load_all_files(true); + for (unsigned int i = 0; i < files_to_open.size(); i++){ + ns_image_server_results_file results(image_server.results_storage.machine_death_times(results_subject,files_to_open[i],"time_path_image_analysis",sql)); + ns_acquire_for_scope tp_i(results.input()); + if (tp_i.is_null()){ + could_load_all_files = false; + continue; + } + death_time_annotation_set.read(annotation_type_to_load,tp_i(),details==ns_machine_analysis_region_data::ns_exclude_fast_moving_animals); + tp_i.release(); + } + if (!could_load_all_files) + return false; + //remove out of bounds data +/* for (unsigned int i = 0 ; i < death_time_annotation_set.size(); i++) + if (death_time_annotation_set.events[i].type == ns_moving_worm_disappearance){ + ns_death_time_annotation a(death_time_annotation_set.events[i]); + std::cerr << a.description() << "\n"; + + }*/ + if (metadata.time_of_last_valid_sample != 0){ + for (ns_death_time_annotation_set::iterator p = death_time_annotation_set.begin(); p != death_time_annotation_set.end();){ +// if(p->stationary_path_id.group_id == 28) +// std::cerr << "MA"; + if (p->time.fully_unbounded()) + throw ns_ex("Fully unbounded interval encountered!"); + if ((!p->time.period_start_was_not_observed && p->time.period_start > metadata.time_of_last_valid_sample)) + p->time.period_start_was_not_observed = true; + if ((!p->time.period_end_was_not_observed && p->time.period_end > metadata.time_of_last_valid_sample)) + p->time.period_end_was_not_observed = true; + if (p->time.fully_unbounded()) + p = death_time_annotation_set.erase(p); + else{ + //events that straddle the externally specified last observation become unbounded + if (p->time.period_end > metadata.time_of_last_valid_sample){ + p->time.period_end_was_not_observed = true; + } + p++; + + } + } + } + + return true; +} + +bool ns_machine_analysis_region_data::recalculate_from_saved_movement_quantification(const ns_64_bit region_id,ns_sql & sql){ + death_time_annotation_set.clear(); + metadata.region_id = region_id; + //load 3d point cloud + + const ns_time_path_solver_parameters solver_parameters(ns_time_path_solver_parameters::default_parameters(region_id,sql)); + const ns_time_series_denoising_parameters time_series_denoising_parameters(ns_time_series_denoising_parameters::load_from_db(region_id,sql)); + ns_time_path_solver solver; + solver.load(region_id,sql); + solver.solve(solver_parameters,time_path_solution,&sql); + time_path_solution.save_to_db(region_id,sql); + //time_path_solution.load_from_db(region_id,sql); + + ns_image_server::ns_posture_analysis_model_cache::const_handle_t posture_analysis_model_handle; + image_server.get_posture_analysis_model_for_region(region_id, posture_analysis_model_handle, sql); + //load cached movement quantification + ns_acquire_for_scope death_time_estimator( + ns_get_death_time_estimator_from_posture_analysis_model(posture_analysis_model_handle().model_specification)); + + time_path_image_analyzer->load_completed_analysis(region_id,time_path_solution,time_series_denoising_parameters, &death_time_estimator(),sql); + death_time_estimator.release(); + //generate annotations from quantification + + time_path_image_analyzer->produce_death_time_annotations(death_time_annotation_set); + + time_path_image_analyzer->clear_annotations(); + //save annotations + ns_image_server_results_subject results_subject; + results_subject.region_id = region_id; + ns_image_server_results_file censoring_results(image_server.results_storage.machine_death_times(results_subject,ns_image_server_results_storage::ns_censoring_and_movement_transitions, + "time_path_image_analysis",sql)); + ns_image_server_results_file state_results(image_server.results_storage.machine_death_times(results_subject,ns_image_server_results_storage::ns_worm_position_annotations, + "time_path_image_analysis",sql)); + ns_acquire_for_scope censoring_out(censoring_results.output()); + ns_acquire_for_scope state_out(state_results.output()); + death_time_annotation_set.write_split_file_column_format(censoring_out(),state_out()); + censoring_out.release(); + state_out.release(); + return true; +} + +void ns_machine_analysis_sample_data::load(const ns_death_time_annotation_set::ns_annotation_type_to_load & annotation_type_to_load,const ns_64_bit sample_id, const ns_region_metadata & sample_metadata,ns_sql & sql, + const ns_64_bit specific_region_id, const bool include_excluded_regions, const ns_machine_analysis_region_data::ns_loading_details & loading_details){ + bool calculate_missing_data = false; + device_name_ = sample_metadata.device; + ns_sql_result reg; + sql << "SELECT r.id FROM sample_region_image_info as r WHERE r.sample_id = " << sample_id << " AND r.censored=0 "; + if (!include_excluded_regions) + sql << " AND r.excluded_from_analysis=0"; + if (specific_region_id!=0) + sql << " AND r.id = " << specific_region_id; + sql << " ORDER BY r.name"; + + sql.get_rows(reg); + if (reg.empty() && specific_region_id!=0) + throw ns_ex("Could not identify region ") << specific_region_id << ". Was it excluded?"; + regions.reserve(reg.size()); + for (unsigned int i = 0; i < reg.size(); i++){ + try{ + const unsigned int s = regions.size(); + regions.resize(s+1); + regions[s] = new ns_machine_analysis_region_data; + ns_64_bit region_id = ns_atoi64(reg[i][0].c_str()); + regions[s]->metadata = sample_metadata; + regions[s]->metadata.load_only_region_info_from_db(region_id,"",sql); + regions[s]->metadata.technique = "Lifespan Machine"; + regions[s]->load_from_db(annotation_type_to_load,loading_details,region_id,sql); + //break; + } + catch(ns_ex & ex){ + std::cerr << (*regions.rbegin())->metadata.sample_name << "::" << (*regions.rbegin())->metadata.region_name << ": " << ex.text() << "\n"; + regions.pop_back(); + } + } + sample_name_ = sample_metadata.sample_name; + sample_id_ = sample_id; +} + +void ns_machine_analysis_data_loader::load_just_survival(ns_lifespan_experiment_set & set,const ns_64_bit region_id, ns_64_bit sample_id, ns_64_bit experiment_id_a, ns_sql & sql, const bool load_excluded_regions, const bool load_by_hand_data){ + set_up_spec_to_load(region_id,sample_id,experiment_id_a,sql,load_excluded_regions); + + ns_region_metadata metadata; + metadata.clear(); + metadata.experiment_name = experiment_name_; + metadata.experiment_id = experiment_id_a; + + std::cerr << "Loading Surival Information\n"; + + ns_genotype_fetcher genotypes; + genotypes.load_from_db(&sql); + //load in just the sample information, one sample at a time to reduce memory consumption + long last_r(-10); + for (unsigned int i = 0; i < samples.size(); i++){ + long r = (100 * i) / samples.size(); + if (r - last_r >= 10) { + std::cout << r << "%..."; + last_r = r; + } + metadata.load_only_sample_info_from_db(samples[i].id(),sql); + samples[i].load(ns_death_time_annotation_set::ns_censoring_and_movement_transitions,metadata.sample_id,metadata,sql,region_id,load_excluded_regions); + ns_death_time_annotation_compiler compiler; + for (unsigned int j = 0 ; j < samples[i].regions.size(); j++){ + samples[i].regions[j]->metadata.genotype = genotypes.genotype_from_strain(samples[i].regions[j]->metadata.strain,&sql); + compiler.add(samples[i].regions[j]->death_time_annotation_set,samples[i].regions[j]->metadata); + if (load_by_hand_data){ + ns_hand_annotation_loader hand_loader; + hand_loader.load_region_annotations(ns_death_time_annotation_set::ns_censoring_and_movement_transitions,samples[i].regions[j]->metadata.region_id,sql); + compiler.add(hand_loader.annotations); + } + } + set.curves.reserve(set.curves.size()+compiler.regions.size()); + for (ns_death_time_annotation_compiler::ns_region_list::iterator p = compiler.regions.begin(); p != compiler.regions.end(); p++){ + const unsigned long s(set.curves.size()); + set.curves.resize(s+1); + p->second.generate_survival_curve(set.curves[s],ns_death_time_annotation::ns_machine_annotations_if_no_by_hand,true,false); + } + samples[i].clear(); + + } + std::cerr << "\n"; +} +void ns_machine_analysis_data_loader::load(const ns_death_time_annotation_set::ns_annotation_type_to_load & annotation_types_to_load,const ns_64_bit region_id, ns_64_bit sample_id, ns_64_bit experiment_id_a, ns_sql & sql, + const bool load_excluded_regions, const ns_machine_analysis_region_data::ns_loading_details & details){ + set_up_spec_to_load(region_id,sample_id,experiment_id_a,sql,load_excluded_regions); + + ns_region_metadata metadata; + metadata.clear(); + metadata.experiment_name = experiment_name_; + metadata.experiment_id = experiment_id_a; + total_number_of_regions_ = 0; + if(!be_quiet) + std::cerr << "Loading " << ns_death_time_annotation_set::annotation_types_to_string(annotation_types_to_load) << "\n"; + + ns_genotype_fetcher genotypes; + genotypes.load_from_db(&sql); + long last_r(-10); + for (unsigned int i = 0; i < samples.size(); i++){ + if (!be_quiet) { + long r = (100 * i) / samples.size(); + if (r - last_r >= 10) { + std::cout << r << "%..."; + last_r = r; + } + } + metadata.load_only_sample_info_from_db(samples[i].id(),sql); + // if (metadata.sample_name != "frog_c") + // continue; + samples[i].load(annotation_types_to_load,metadata.sample_id,metadata,sql,region_id,load_excluded_regions,details); + for (unsigned int j = 0 ; j < samples[i].regions.size(); j++) + samples[i].regions[j]->metadata.genotype = genotypes.genotype_from_strain(samples[i].regions[j]->metadata.strain,&sql); + total_number_of_regions_+=samples[i].regions.size(); + // break; + } + if(!be_quiet) + std::cerr << "\n"; +} +void ns_machine_analysis_data_loader::set_up_spec_to_load(const ns_64_bit & region_id, ns_64_bit & sample_id, ns_64_bit & experiment_id_a, ns_sql & sql, const bool load_excluded_regions){ + const bool region_specified(region_id != 0); + const bool sample_specified(sample_id != 0); + if (region_id == 0 && sample_id == 0 && experiment_id_a==0) + throw ns_ex("No data requested!"); + + if (region_id != 0){ + sql << "SELECT sample_id FROM sample_region_image_info WHERE id = " << region_id; + ns_sql_result res; + sql.get_rows(res); + if (res.size() == 0) + throw ns_ex("ns_experiment_movement_results::load()::Could not load region information ") << region_id; + sample_id = ns_atoi64(res[0][0].c_str()); + } + if (sample_id != 0){ + sql << "SELECT experiment_id FROM capture_samples WHERE id = " << sample_id; + ns_sql_result res; + sql.get_rows(res); + if (res.size() == 0) + throw ns_ex("ns_experiment_movement_results::load()::Could not load sample information ") << sample_id; + experiment_id_a = ns_atoi64(res[0][0].c_str()); + } + + sql << "SELECT name FROM experiments WHERE id=" << experiment_id_a; + ns_sql_result res; + sql.get_rows(res); + + if (res.size() == 0) + throw ns_ex("ns_experiment_movement_results::load()::Could not load experiment id=") << experiment_id_a; + + experiment_name_ = res[0][0]; + experiment_id_ = experiment_id_a; + std::vector sample_ids; + + if (!region_specified && !sample_specified){ + sql << "SELECT id FROM capture_samples WHERE censored=0 AND experiment_id = " << experiment_id_a; + if (sample_id != 0) + sql << " AND id = " << sample_id; + ns_sql_result samp; + sql.get_rows(samp); + samples.resize(samp.size()); + for (unsigned int i = 0; i < samp.size(); i++) + samples[i].set_id(ns_atoi64(samp[i][0].c_str())); + } + else{ + //add just the sample + samples.resize(1,sample_id); + } + +} + + +ns_time_series_denoising_parameters ns_time_series_denoising_parameters::load_from_db(const ns_64_bit region_id, ns_sql & sql){ + sql << "SELECT time_series_denoising_flag FROM sample_region_image_info WHERE id = " << region_id; + ns_sql_result res; + sql.get_rows(res); + if (res.size() == 0) + throw ns_ex("ns_time_series_denoising_parameters::load_from_db()::Could not find region ") << region_id << " in db"; + ns_time_series_denoising_parameters p; + p.movement_score_normalization = (ns_time_series_denoising_parameters::ns_movement_score_normalization_type)atol(res[0][0].c_str()); + + return p; +} \ No newline at end of file diff --git a/ns_image_server/movement_analysis/ns_machine_analysis_data_loader.h b/ns_image_server/movement_analysis/ns_machine_analysis_data_loader.h new file mode 100644 index 000000000..53d45f6d2 --- /dev/null +++ b/ns_image_server/movement_analysis/ns_machine_analysis_data_loader.h @@ -0,0 +1,95 @@ +#ifndef NS_MACHINE_ANALYSIS_DATA_LOADER +#define NS_MACHINE_ANALYSIS_DATA_LOADER +#include "ns_sql.h" +#include "ns_death_time_annotation_set.h" +#include "ns_movement_measurement.h" +#include "ns_time_path_image_analyzer.h" +#include + +class ns_machine_analysis_region_data{ +private: + //disallow copy constructor + ns_machine_analysis_region_data(const ns_machine_analysis_region_data &); +public: + + typedef enum {ns_load_all,ns_exclude_fast_moving_animals} ns_loading_details; + + ns_machine_analysis_region_data():summary_series_generated_(false), time_path_image_analyzer(new ns_time_path_image_movement_analyzer){} + ~ns_machine_analysis_region_data() { + ns_safe_delete(time_path_image_analyzer); + } + ns_death_time_annotation_set death_time_annotation_set; + mutable ns_region_metadata metadata; + void calculate_survival(){} + const ns_worm_movement_summary_series & summary_series(const ns_death_time_annotation::ns_by_hand_annotation_integration_strategy & by_hand_strategy_,const ns_death_time_annotation::ns_multiworm_censoring_strategy & cs, + ns_death_time_annotation::ns_missing_worm_return_strategy & css, const ns_animals_that_slow_but_do_not_die_handling_strategy & sls) const{ + if (!summary_series_generated_){ + ns_death_time_annotation_compiler c; + c.add(death_time_annotation_set); + cached_summary_series.from_death_time_annotations(by_hand_strategy_,cs,css,c,sls); + summary_series_generated_=true; + } + return cached_summary_series; + } + + bool recalculate_from_saved_movement_quantification(const ns_64_bit region_id,ns_sql & sql); + + bool load_from_db(const ns_death_time_annotation_set::ns_annotation_type_to_load & annotation_types_to_load,const ns_loading_details & details, + const ns_64_bit region_id,ns_sql & sql); + + ns_time_path_solution time_path_solution; + ns_time_path_image_movement_analyzer * time_path_image_analyzer; + +private: + mutable ns_worm_movement_summary_series cached_summary_series; + mutable bool summary_series_generated_; + //ns_time_path_image_movement_analyzer time_path_image_analyzer; + //ns_time_path_solution time_path_solution; +}; + +class ns_machine_analysis_sample_data{ +public: + void load(const ns_death_time_annotation_set::ns_annotation_type_to_load & annotation_types_to_load, + const ns_64_bit sample_id, const ns_region_metadata & sample_metadata, ns_sql & sql, + const ns_64_bit specific_region_id=0, const bool include_excluded_regions=false, + const ns_machine_analysis_region_data::ns_loading_details & loading_details=ns_machine_analysis_region_data::ns_load_all); + ns_machine_analysis_sample_data(){} + ns_machine_analysis_sample_data(const ns_64_bit id):sample_id_(id){} + std::vector regions; + ns_64_bit id() const {return sample_id_;} + const std::string & name() const {return sample_name_;} + const std::string & device_name() const {return device_name_;} + void set_id(const ns_64_bit id){sample_id_ =id;} + void clear() { sample_id_ = 0; device_name_.clear(); sample_name_.clear(); for (unsigned int i = 0; i < regions.size(); i++) ns_safe_delete(regions[i]); regions.clear(); } +private: + std::string device_name_; + std::string sample_name_; + ns_64_bit sample_id_; +}; + +class ns_machine_analysis_data_loader{ +public: + void clear(){ + samples.resize(0); + experiment_id_ = 0; + experiment_name_.resize(0); + total_number_of_regions_ = 0; + } + ns_machine_analysis_data_loader(const bool be_quiet_=false):be_quiet(be_quiet_){} + unsigned long total_number_of_regions()const{return total_number_of_regions_;} + void load(const ns_death_time_annotation_set::ns_annotation_type_to_load & annotation_types_to_load,const ns_64_bit region_id, ns_64_bit sample_id, ns_64_bit experiment_id_a, ns_sql & sql, + const bool include_excluded_regions=false, const ns_machine_analysis_region_data::ns_loading_details & details=ns_machine_analysis_region_data::ns_load_all); + //uses much less memory just to compile the data into a survival curve data set + void load_just_survival(ns_lifespan_experiment_set & set,ns_64_bit region_id, ns_64_bit sample_id, ns_64_bit experiment_id_a, ns_sql & sql, const bool load_excluded_regions, const bool load_by_hand_data); + const std::string & experiment_name(){return experiment_name_;} + const ns_64_bit experiment_id(){return experiment_id_;} + std::vector samples; +private: + bool be_quiet; + void set_up_spec_to_load(const ns_64_bit & region_id, ns_64_bit &sample_id, ns_64_bit & experiment_id_a, ns_sql & sql, const bool load_excluded_regions); + std::string experiment_name_; + ns_64_bit experiment_id_; + unsigned long total_number_of_regions_; +}; + +#endif