Merge branch 'rh/disable_mux_change_trim_r9' into 'master'

DOR-607 Disable mux change trimming on r9.4.1 data

See merge request machine-learning/dorado!899

dorado/data_loader/DataLoader.cpp

@@ -3,6 +3,7 @@
 #include "models/kits.h"
 #include "models/models.h"
 #include "read_pipeline/ReadPipeline.h"
+#include "read_pipeline/messages.h"
 #include "utils/PostCondition.h"
 #include "utils/time_utils.h"
 #include "utils/types.h"
@@ -229,6 +230,7 @@ SimplexReadPtr process_pod5_read(
     // used has a rapid adapter and which one.
     const auto condition_info = models::ConditionInfo(get_chemistry_key(run_info_data));
     new_read->read_common.rapid_chemistry = condition_info.rapid_chemistry();
+    new_read->read_common.chemistry = condition_info.chemistry();
 
     pod5_end_reason_t end_reason_value{POD5_END_REASON_UNKNOWN};
     char end_reason_string_value[200];
@@ -853,6 +855,7 @@ void DataLoader::load_pod5_reads_from_file_by_read_ids(const std::string& path,
 
         for (auto& v : futures) {
             auto read = v.get();
+            check_read(read);
             m_pipeline.push_message(std::move(read));
             m_loaded_read_count++;
         }
@@ -911,6 +914,7 @@ void DataLoader::load_pod5_reads_from_file(const std::string& path) {
 
         for (auto& v : futures) {
             auto read = v.get();
+            check_read(read);
             m_pipeline.push_message(std::move(read));
             m_loaded_read_count++;
         }
@@ -1023,6 +1027,15 @@ void DataLoader::load_fast5_reads_from_file(const std::string& path) {
     }
 }
 
+void DataLoader::check_read(const SimplexReadPtr& read) {
+    if (read->read_common.chemistry == models::Chemistry::UNKNOWN &&
+        m_log_unknown_chemistry.exchange(false)) {
+        spdlog::warn(
+                "Could not determine sequencing Chemistry from read data - "
+                "some features might be disabled");
+    }
+}
+
 DataLoader::DataLoader(Pipeline& pipeline,
                        const std::string& device,
                        size_t num_worker_threads,

dorado/data_loader/DataLoader.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "models/models.h"
+#include "read_pipeline/messages.h"
 #include "utils/stats.h"
 #include "utils/types.h"
 
@@ -75,6 +76,7 @@ class DataLoader {
     void load_pod5_reads_from_file_by_read_ids(const std::string& path,
                                                const std::vector<ReadID>& read_ids);
     void load_read_channels(std::filesystem::path data_path, bool recursive_file_loading);
+
     Pipeline& m_pipeline;  // Where should the loaded reads go?
     std::atomic<size_t> m_loaded_read_count{0};
     std::string m_device;
@@ -87,6 +89,11 @@ class DataLoader {
     std::unordered_map<int, std::vector<ReadSortInfo>> m_reads_by_channel;
     std::unordered_map<std::string, size_t> m_read_id_to_index;
     int m_max_channel{0};
+
+    // Issue warnings if read is potentially problematic
+    inline void check_read(const SimplexReadPtr& read);
+    // A flag to warn only once if the data chemsitry is known
+    std::atomic<bool> m_log_unknown_chemistry{true};
 };
 
 }  // namespace dorado

dorado/read_pipeline/BasecallerNode.cpp

@@ -2,6 +2,7 @@
 
 #include "basecall/CRFModelConfig.h"
 #include "basecall/ModelRunnerBase.h"
+#include "models/kits.h"
 #include "read_utils.h"
 #include "stitch.h"
 #include "utils/stats.h"
@@ -204,8 +205,12 @@ void BasecallerNode::working_reads_manager() {
             // Chunks have ownership of the working read, so destroy them to avoid a leak.
             working_read->called_chunks.clear();
 
-            // Trim reads which are affected by mux change and unblocking
-            utils::mux_change_trim_read(read_common_data);
+            // Do not trim R9.4.1 data to avoid changes to legacy products
+            // Check here to avoid adding models lib as a dependency of utils
+            if (read_common_data.chemistry != models::Chemistry::DNA_R9_4_1_E8) {
+                // Trim reads which are affected by mux change and unblocking
+                utils::mux_change_trim_read(read_common_data);
+            }
 
             // Cleanup the working read.
             {

dorado/read_pipeline/messages.h

@@ -102,6 +102,8 @@ class ReadCommon {
     // `True` if the basecall model is an RNA model
     bool is_rna_model{false};
 
+    // The chemistry type if any
+    models::Chemistry chemistry{models::Chemistry::UNKNOWN};
     // The rapid chemistry adapter type if any - sourced from the read info data
     models::RapidChemistry rapid_chemistry{models::RapidChemistry::UNKNOWN};