Intermediate check-in for the Read Cache functionality.

docs/manual/html/classmdf_1_1_i_channel_observer-members.html

@@ -100,7 +100,7 @@
   <tr bgcolor="#f0f0f0" class="even"><td class="entry"><b>IChannelObserver</b>()=delete (defined in <a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a>)</td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"></td></tr>
   <tr bgcolor="#f0f0f0" class="odd"><td class="entry"><b>IChannelObserver</b>(const IChannelObserver &amp;)=delete (defined in <a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a>)</td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"></td></tr>
   <tr bgcolor="#f0f0f0" class="even"><td class="entry"><b>IChannelObserver</b>(IChannelObserver &amp;&amp;)=delete (defined in <a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a>)</td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"></td></tr>
-  <tr class="odd"><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">index_list_</a></td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"><span class="mlabel">protected</span></td></tr>
+  <tr class="odd"><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">offset_list_</a></td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"><span class="mlabel">protected</span></td></tr>
   <tr class="even"><td class="entry"><a class="el" href="classmdf_1_1_i_sample_observer.html#a217984e2da0d4f94df25473bb7045ebb">ISampleObserver</a>()=default</td><td class="entry"><a class="el" href="classmdf_1_1_i_sample_observer.html">mdf::ISampleObserver</a></td><td class="entry"></td></tr>
   <tr class="odd"><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html#a05bd6a3819c779ca9b326f1eefbfd685">IsArray</a>() const</td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"></td></tr>
   <tr class="even"><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html#a6d5a1e2d4a99e8068231848f9ffaa8fe">IsMaster</a>() const</td><td class="entry"><a class="el" href="classmdf_1_1_i_channel_observer.html">mdf::IChannelObserver</a></td><td class="entry"></td></tr>

docs/manual/html/classmdf_1_1_i_channel_observer.html

@@ -231,7 +231,7 @@
 <tr class="memdesc:a1054f169e39eeeda91cee4ecf6e691f6"><td class="mdescLeft">&#160;</td><td class="mdescRight">Reference to the channel (CN) block. <br /></td></tr>
 <tr class="separator:a1054f169e39eeeda91cee4ecf6e691f6"><td class="memSeparator" colspan="2">&#160;</td></tr>
 <tr class="memitem:a866292eb811a5384c482de43d1b360c7" id="r_a866292eb811a5384c482de43d1b360c7"><td class="memItemLeft" align="right" valign="top"><a id="a866292eb811a5384c482de43d1b360c7" name="a866292eb811a5384c482de43d1b360c7"></a>
-std::vector&lt; uint64_t &gt;&#160;</td><td class="memItemRight" valign="bottom"><b>index_list_</b></td></tr>
+std::vector&lt; uint64_t &gt;&#160;</td><td class="memItemRight" valign="bottom"><b>offset_list_</b></td></tr>
 <tr class="memdesc:a866292eb811a5384c482de43d1b360c7"><td class="mdescLeft">&#160;</td><td class="mdescRight">Only used for VLSD channels. <br /></td></tr>
 <tr class="separator:a866292eb811a5384c482de43d1b360c7"><td class="memSeparator" colspan="2">&#160;</td></tr>
 </table>

docs/manual/html/functions_i.html

@@ -78,7 +78,7 @@ <h3><a id="index_i" name="index_i"></a>- i -</h3><ul>
 <li>IncrementNofSamples()&#160;:&#160;<a class="el" href="classmdf_1_1_mdf_writer.html#ac750827df8d944250173df2528f0dcfb">mdf::MdfWriter</a></li>
 <li>IncrementSample()&#160;:&#160;<a class="el" href="classmdf_1_1_i_channel_group.html#a5a886201c6de6c9bcaab954beb097f1c">mdf::IChannelGroup</a></li>
 <li>Index()&#160;:&#160;<a class="el" href="classmdf_1_1_i_block.html#a1d4238fc723d16ccd8e010d098d4f45f">mdf::IBlock</a>, <a class="el" href="classmdf_1_1_mdf_reader.html#af0675cacfb831f8700c9c323feb42ef8">mdf::MdfReader</a></li>
-<li>index_list_&#160;:&#160;<a class="el" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">mdf::IChannelObserver</a></li>
+<li>offset_list_&#160;:&#160;<a class="el" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">mdf::IChannelObserver</a></li>
 <li>Init()&#160;:&#160;<a class="el" href="classmdf_1_1_mdf_writer.html#a77e457f8f1b7d048a7b575b94a4f3b2f">mdf::MdfWriter</a></li>
 <li>InitMd()&#160;:&#160;<a class="el" href="classmdf_1_1_i_meta_data.html#a149c5c7f67976bd281e881fb875d614f">mdf::IMetaData</a></li>
 <li>InitMeasurement()&#160;:&#160;<a class="el" href="classmdf_1_1_mdf_writer.html#a99e1aa5229ebbee9ee533e6a9d12445c">mdf::MdfWriter</a></li>

docs/manual/html/functions_vars.html

@@ -105,7 +105,7 @@ <h3><a id="index_f" name="index_f"></a>- f -</h3><ul>
 
 
 <h3><a id="index_i" name="index_i"></a>- i -</h3><ul>
-<li>index_list_&#160;:&#160;<a class="el" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">mdf::IChannelObserver</a></li>
+<li>offset_list_&#160;:&#160;<a class="el" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">mdf::IChannelObserver</a></li>
 <li>input_quantity_list_&#160;:&#160;<a class="el" href="classmdf_1_1_i_channel_array.html#ad726a5ffaae4be9490a3ab7605487410">mdf::IChannelArray</a></li>
 </ul>
 

docs/manual/html/ichannelobserver_8h_source.html

@@ -104,7 +104,7 @@
 <div class="line"><a id="l00027" name="l00027"></a><span class="lineno">   27</span> <span class="keyword">protected</span>:</div>
 <div class="line"><a id="l00028" name="l00028"></a><span class="lineno"><a class="line" href="classmdf_1_1_i_channel_observer.html#a1054f169e39eeeda91cee4ecf6e691f6">   28</a></span>   <span class="keyword">const</span> <a class="code hl_class" href="classmdf_1_1_i_channel.html">IChannel</a>&amp; <a class="code hl_variable" href="classmdf_1_1_i_channel_observer.html#a1054f169e39eeeda91cee4ecf6e691f6">channel_</a>; </div>
 <div class="line"><a id="l00029" name="l00029"></a><span class="lineno">   29</span> </div>
-<div class="line"><a id="l00030" name="l00030"></a><span class="lineno"><a class="line" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">   30</a></span>  std::vector&lt;uint64_t&gt; <a class="code hl_variable" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">index_list_</a>; </div>
+<div class="line"><a id="l00030" name="l00030"></a><span class="lineno"><a class="line" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">   30</a></span>  std::vector&lt;uint64_t&gt; <a class="code hl_variable" href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">offset_list_</a>; </div>
 <div class="line"><a id="l00031" name="l00031"></a><span class="lineno"><a class="line" href="classmdf_1_1_i_channel_observer.html#a119853914104987a7f78a9d4995980e7">   31</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">bool</span> <a class="code hl_function" href="classmdf_1_1_i_channel_observer.html#a119853914104987a7f78a9d4995980e7">GetSampleUnsigned</a>(uint64_t sample, uint64_t&amp; value, uint64_t array_index)</div>
 <div class="line"><a id="l00032" name="l00032"></a><span class="lineno">   32</span>      <span class="keyword">const</span> = 0; </div>
 <div class="line"><a id="l00033" name="l00033"></a><span class="lineno"><a class="line" href="classmdf_1_1_i_channel_observer.html#a77c482deccfe18ed5b240207103e0d71">   33</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">bool</span> <a class="code hl_function" href="classmdf_1_1_i_channel_observer.html#a77c482deccfe18ed5b240207103e0d71">GetSampleSigned</a>(uint64_t sample, int64_t&amp; value, uint64_t array_index)</div>
@@ -287,7 +287,7 @@
 <div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_a70c8fa4fc52f464bf33487dde3c8e493"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#a70c8fa4fc52f464bf33487dde3c8e493">mdf::IChannelObserver::GetSampleByteArray</a></div><div class="ttdeci">virtual bool GetSampleByteArray(uint64_t sample, std::vector&lt; uint8_t &gt; &amp;value) const =0</div><div class="ttdoc">Returns a byte array sample value.</div></div>
 <div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_a77c482deccfe18ed5b240207103e0d71"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#a77c482deccfe18ed5b240207103e0d71">mdf::IChannelObserver::GetSampleSigned</a></div><div class="ttdeci">virtual bool GetSampleSigned(uint64_t sample, int64_t &amp;value, uint64_t array_index) const =0</div><div class="ttdoc">Returns a signed sample value.</div></div>
 <div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_a819be1e44a252c08af1ae72b0815adba"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#a819be1e44a252c08af1ae72b0815adba">mdf::IChannelObserver::EngValueToString</a></div><div class="ttdeci">std::string EngValueToString(uint64_t sample) const</div><div class="ttdoc">Support function that convert a sample value to a user friendly string.</div></div>
-<div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_a866292eb811a5384c482de43d1b360c7"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">mdf::IChannelObserver::index_list_</a></div><div class="ttdeci">std::vector&lt; uint64_t &gt; index_list_</div><div class="ttdoc">Only used for VLSD channels.</div><div class="ttdef"><b>Definition</b> ichannelobserver.h:30</div></div>
+<div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_a866292eb811a5384c482de43d1b360c7"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#a866292eb811a5384c482de43d1b360c7">mdf::IChannelObserver::offset_list_</a></div><div class="ttdeci">std::vector&lt; uint64_t &gt; offset_list_</div><div class="ttdoc">Only used for VLSD channels.</div><div class="ttdef"><b>Definition</b> ichannelobserver.h:30</div></div>
 <div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_a944b4093b9e83896b2687a87618b787e"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#a944b4093b9e83896b2687a87618b787e">mdf::IChannelObserver::Unit</a></div><div class="ttdeci">std::string Unit() const</div><div class="ttdoc">Channel unit.</div></div>
 <div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_ad6611bae034a767205da137b93bbf3aa"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#ad6611bae034a767205da137b93bbf3aa">mdf::IChannelObserver::NofSamples</a></div><div class="ttdeci">virtual uint64_t NofSamples() const =0</div><div class="ttdoc">Returns number of samples.</div></div>
 <div class="ttc" id="aclassmdf_1_1_i_channel_observer_html_ae5b6e85bdf1a75b68d1c84a49c56da60"><div class="ttname"><a href="classmdf_1_1_i_channel_observer.html#ae5b6e85bdf1a75b68d1c84a49c56da60">mdf::IChannelObserver::ArraySize</a></div><div class="ttdeci">uint64_t ArraySize() const</div><div class="ttdoc">If this is an array channel, this function returns the array size.</div></div>

include/mdf/ichannelobserver.h

@@ -26,8 +26,11 @@ namespace mdf {
 class IChannelObserver : public ISampleObserver {
  protected:
    const IChannel& channel_; ///< Reference to the channel (CN) block.
+   bool read_vlsd_data_ = true;
+
+  std::vector<uint64_t> offset_list_; ///< Only used for VLSD channels.
+  std::vector<bool> valid_list_; ///< List of valid samples.
 
-  std::vector<uint64_t> index_list_; ///< Only used for VLSD channels.
   virtual bool GetSampleUnsigned(uint64_t sample, uint64_t& value, uint64_t array_index)
       const = 0; ///< Returns a unsigned  sample value.
   virtual bool GetSampleSigned(uint64_t sample, int64_t& value, uint64_t array_index)
@@ -65,8 +68,14 @@ class IChannelObserver : public ISampleObserver {
    * @return Returns the channel object.
    */
   [[nodiscard]] const IChannel& Channel() const;
+
   [[nodiscard]] bool IsMaster() const; ///< True if this is the master channel.
+
   [[nodiscard]] bool IsArray() const; ///< True if this channel is an array channel.
+
+  void ReadVlsdData(bool read_vlsd_data);
+  [[nodiscard]] bool ReadVlsdData() const { return read_vlsd_data_; }
+
   /** \brief If this is an array channel, this function returns the array size.
    *
    * Returns the array size if the channel is an array channel. The function
@@ -105,6 +114,8 @@ class IChannelObserver : public ISampleObserver {
    * @return JSON formatted string
    */
   [[nodiscard]] std::string EngValueToString(uint64_t sample) const;
+
+   bool GetOffsetValue(uint64_t sample, uint64_t& offset) const;
 };
 
 template <typename V>

include/mdf/idatagroup.h

@@ -111,8 +111,14 @@ class IDataGroup : public IBlock {
   [[nodiscard]] virtual IChannelGroup *FindParentChannelGroup(
       const IChannel &channel) const = 0;
 
+  /**
+   * \brief Checks if this data group subscribes on a specific record.
+   * @param record_id Record ID of the channel group
+   * @return True if the observer list subscribe on this channel group.
+   */
+  [[nodiscard]] bool IsSubscribingOnRecord(uint64_t record_id) const;
  protected:
-  mutable std::vector<ISampleObserver*> observer_list; ///< List of observers.
+  mutable std::vector<ISampleObserver*> observer_list_; ///< List of observers.
   ~IDataGroup() override = default; ///< Default destructor
 
  private:

include/mdf/isampleobserver.h

@@ -9,6 +9,7 @@
 #pragma once
 #include <cstdint>
 #include <vector>
+#include <set>
 #include <functional>
 
 #include "mdf/ichannelgroup.h"
@@ -54,6 +55,17 @@ class ISampleObserver {
   virtual void OnSample(uint64_t sample, uint64_t record_id,
                         const std::vector<uint8_t>& record);
 
+  /**
+   * \brief Function that test if this observer needs to read a specific
+   * record.
+   *
+   * @param record_id The channel groups record ID
+   * @return True if this channel subscribe on this record (channel group).
+   */
+  [[nodiscard]] bool IsRecordIdNeeded(uint64_t record_id) const {
+    return record_id_list_.find(record_id) != record_id_list_.cend();
+  }
+
   /** \brief Function object that is called if assigned.
    *
    * The function object is typically assigned to a lambda function.
@@ -171,7 +183,7 @@ class ISampleObserver {
   }
 
  protected:
-
+  std::set<uint64_t> record_id_list_; ///< List of subscribed channel groups.
   const IDataGroup& data_group_;  ///< Reference to the data group (DG) block.
  private:
   bool attached_ = false; ///< True if the observer is active

mdflib/CMakeLists.txt

@@ -19,7 +19,7 @@ add_library(mdf STATIC
         src/cc3block.cpp src/cc3block.h
         src/cd3block.cpp src/cd3block.h
         src/ce3block.cpp src/ce3block.h
-        ../include/mdf/mdfreader.h src/mdfreader.cpp
+        src/mdfreader.cpp ../include/mdf/mdfreader.h
         src/mdf3file.cpp src/mdf3file.h
         src/hd4block.cpp src/hd4block.h
         src/tx4block.cpp src/tx4block.h
@@ -46,11 +46,10 @@ add_library(mdf STATIC
         src/datalistblock.cpp src/datalistblock.h
         src/blockproperty.cpp src/blockproperty.h
         src/mdf4timestamp.cpp src/mdf4timestamp.h
-        ../include/mdf/iattachment.h src/iattachment.cpp
+        src/iattachment.cpp ../include/mdf/iattachment.h
         ../include/mdf/ichannel.h src/ichannel.cpp
         ../include/mdf/idatagroup.h src/idatagroup.cpp
         ../include/mdf/ichannelgroup.h src/ichannelgroup.cpp
-        ../include/mdf/isampleobserver.h
         src/channelobserver.h src/channelobserver.cpp
         ../include/mdf/ichannelobserver.h src/ichannelobserver.cpp
         ../include/mdf/ichannelconversion.h src/ichannelconversion.cpp
@@ -89,11 +88,10 @@ add_library(mdf STATIC
         src/vlsddata.cpp src/vlsddata.h
         src/mdfbuslogger.cpp src/mdfbuslogger.h
         src/canmessage.cpp ../include/mdf/canmessage.h
-        src/dbchelper.cpp
-        src/dbchelper.h
-        src/isampleobserver.cpp
-        src/isamplereduction.cpp
-        ../include/mdf/isamplereduction.h
+        src/dbchelper.cpp src/dbchelper.h
+        src/isampleobserver.cpp  ../include/mdf/isampleobserver.h
+        src/isamplereduction.cpp ../include/mdf/isamplereduction.h
+        src/readcache.cpp src/readcache.h
 )
 
 if(VCPKG)

mdflib/src/channelobserver.h

@@ -18,7 +18,7 @@ class ChannelObserver : public IChannelObserver {
  private:
   uint64_t record_id_ = 0;
   std::vector<T> value_list_;
-  std::vector<bool> valid_list_;
+
   const IChannelGroup& group_;       ///< Reference to the channel group (CG) block.
 
 
@@ -48,7 +48,7 @@ class ChannelObserver : public IChannelObserver {
     value_list_.resize(group.NofSamples() * array_size, T{});
 
     if (channel_.Type() == ChannelType::VariableLength) {
-      index_list_.resize(group.NofSamples() * array_size,0);
+      offset_list_.resize(group.NofSamples() * array_size, 0);
     }
     ChannelObserver::AttachObserver();
   }
@@ -101,25 +101,30 @@ class ChannelObserver : public IChannelObserver {
             uint64_t offset = 0; // Offset into SD/CG block
             valid = channel_.GetUnsignedValue(record, offset, array_index);
 
-            const auto sample_index = (sample * array_size) + array_index;
-            if (sample_index < index_list_.size()) {
-              index_list_[sample_index] = offset;
+            const auto sample_index = static_cast<size_t>((sample * array_size) + array_index);
+            if (sample_index < offset_list_.size()) {
+              offset_list_[sample_index] = offset;
             }
-            // Value should be in the channels data list (SD). The channels
-            // GetChannelValue handle this situation
-            valid = channel_.GetChannelValue(record, value, array_index);
             if (sample_index < valid_list_.size()) {
               valid_list_[sample_index] = valid;
             }
-            if (sample_index < value_list_.size()) {
-              value_list_[sample_index] = value;
+            if (ReadVlsdData()) {
+              // Value should be in the channels data list (SD). The channels
+              // GetChannelValue handle this situation
+              valid = channel_.GetChannelValue(record, value, array_index);
+              if (sample_index < valid_list_.size()) {
+                valid_list_[sample_index] = valid;
+              }
+              if (sample_index < value_list_.size()) {
+                value_list_[sample_index] = value;
+              }
             }
           }
         } else if (channel_.VlsdRecordId() > 0 &&
                    record_id == channel_.VlsdRecordId()) {
           // Add the VLSD offset data to this channel
           for (uint64_t array_index = 0; array_index < array_size; ++array_index) {
-            const auto sample_index = (sample * array_size) + array_index;
+            const auto sample_index = static_cast<size_t>((sample * array_size) + array_index);
             valid = channel_.GetChannelValue(record, value, array_index);
             if (sample_index < value_list_.size()) {
               value_list_[sample_index] = value;
@@ -138,7 +143,7 @@ class ChannelObserver : public IChannelObserver {
       default:
         if (record_id_ == record_id) {
           for (uint64_t array_index = 0; array_index < array_size; ++array_index) {
-            const auto sample_index = (sample * array_size) + array_index;
+            const auto sample_index = static_cast<size_t>((sample * array_size) + array_index);
             valid = channel_.GetChannelValue(record, value, array_index);
             if (sample_index < value_list_.size()) {
               value_list_[sample_index] = value;

mdflib/src/datablock.cpp

@@ -48,6 +48,7 @@ size_t DataBlock::CopyDataToBuffer(std::FILE *from_file,
   if (data_size == 0) {
     return 0;
   }
+
   const auto reads =
       fread(buffer.data() + buffer_index, 1, data_size, from_file);
   buffer_index += reads;