(O2-1203) [CTF] use merging/splitting iterators during CTF ecoding/decoding for TPC

DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h

@@ -27,6 +27,25 @@ namespace o2
 {
 namespace ctf
 {
+
+namespace detail
+{
+
+template <class, class Enable = void>
+struct is_iterator : std::false_type {
+};
+
+template <class T>
+struct is_iterator<T, std::enable_if_t<
+                        std::is_base_of_v<std::input_iterator_tag, typename std::iterator_traits<T>::iterator_category> ||
+                        std::is_same_v<std::output_iterator_tag, typename std::iterator_traits<T>::iterator_category>>>
+  : std::true_type {
+};
+
+template <class T>
+inline constexpr bool is_iterator_v = is_iterator<T>::value;
+} // namespace detail
+
 using namespace o2::rans;
 constexpr size_t Alignment = 16;
 
@@ -376,20 +395,20 @@ class EncodedBlocks
   template <typename VE, typename VB>
   inline void encode(const VE& src, int slot, uint8_t probabilityBits, Metadata::OptStore opt, VB* buffer = nullptr, const void* encoderExt = nullptr)
   {
-    encode(&(*src.begin()), &(*src.end()), slot, probabilityBits, opt, buffer, encoderExt);
+    encode(std::begin(src), std::end(src), slot, probabilityBits, opt, buffer, encoderExt);
   }
 
   /// encode vector src to bloc at provided slot
   template <typename S_IT, typename VB>
   void encode(const S_IT srcBegin, const S_IT srcEnd, int slot, uint8_t probabilityBits, Metadata::OptStore opt, VB* buffer = nullptr, const void* encoderExt = nullptr);
 
   /// decode block at provided slot to destination vector (will be resized as needed)
-  template <typename VD>
-  void decode(VD& dest, int slot, const void* decoderExt = nullptr) const;
+  template <class container_T, class container_IT = typename container_T::iterator>
+  void decode(container_T& dest, int slot, const void* decoderExt = nullptr) const;
 
   /// decode block at provided slot to destination pointer, the needed space assumed to be available
-  template <typename D>
-  void decode(D* dest, int slot, const void* decoderExt = nullptr) const;
+  template <typename D_IT, std::enable_if_t<detail::is_iterator_v<D_IT>, bool> = true>
+  void decode(D_IT dest, int slot, const void* decoderExt = nullptr) const;
 
   /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables
   static std::vector<char> createDictionaryBlocks(const std::vector<o2::rans::FrequencyTable>& vfreq, const std::vector<Metadata>& prbits);
@@ -666,39 +685,41 @@ void EncodedBlocks<H, N, W>::print(const std::string& prefix) const
 
 ///_____________________________________________________________________________
 template <typename H, int N, typename W>
-template <typename VD>
-inline void EncodedBlocks<H, N, W>::decode(VD& dest,                     // destination container
+template <class container_T, class container_IT>
+inline void EncodedBlocks<H, N, W>::decode(container_T& dest,            // destination container
                                            int slot,                     // slot of the block to decode
                                            const void* decoderExt) const // optional externally provided decoder
 {
   dest.resize(mMetadata[slot].messageLength); // allocate output buffer
-  decode(dest.data(), slot, decoderExt);
+  decode(std::begin(dest), slot, decoderExt);
 }
 
 ///_____________________________________________________________________________
 template <typename H, int N, typename W>
-template <typename D>
-void EncodedBlocks<H, N, W>::decode(D* dest,                      // destination pointer
+template <typename D_IT, std::enable_if_t<detail::is_iterator_v<D_IT>, bool>>
+void EncodedBlocks<H, N, W>::decode(D_IT dest,                    // iterator to destination
                                     int slot,                     // slot of the block to decode
                                     const void* decoderExt) const // optional externally provided decoder
 {
   // get references to the right data
   const auto& block = mBlocks[slot];
   const auto& md = mMetadata[slot];
 
+  using dest_t = typename std::iterator_traits<D_IT>::value_type;
+
   // decode
   if (block.getNStored()) {
     if (md.opt == Metadata::OptStore::EENCODE) {
       if (!decoderExt && !block.getNDict()) {
         LOG(ERROR) << "Dictionaty is not saved for slot " << slot << " and no external decoder is provided";
         throw std::runtime_error("Dictionary is not saved and no external decoder provided");
       }
-      const o2::rans::LiteralDecoder64<D>* decoder = reinterpret_cast<const o2::rans::LiteralDecoder64<D>*>(decoderExt);
-      std::unique_ptr<o2::rans::LiteralDecoder64<D>> decoderLoc;
+      const o2::rans::LiteralDecoder64<dest_t>* decoder = reinterpret_cast<const o2::rans::LiteralDecoder64<dest_t>*>(decoderExt);
+      std::unique_ptr<o2::rans::LiteralDecoder64<dest_t>> decoderLoc;
       if (block.getNDict()) { // if dictionaty is saved, prefer it
         o2::rans::FrequencyTable frequencies;
         frequencies.addFrequencies(block.getDict(), block.getDict() + block.getNDict(), md.min, md.max);
-        decoderLoc = std::make_unique<o2::rans::LiteralDecoder64<D>>(frequencies, md.probabilityBits);
+        decoderLoc = std::make_unique<o2::rans::LiteralDecoder64<dest_t>>(frequencies, md.probabilityBits);
         decoder = decoderLoc.get();
       } else { // verify that decoded corresponds to stored metadata
         if (md.min != decoder->getMinSymbol() || md.max != decoder->getMaxSymbol()) {
@@ -708,16 +729,20 @@ void EncodedBlocks<H, N, W>::decode(D* dest,                      // destination
         }
       }
       // load incompressible symbols if they existed
-      std::vector<D> literals;
+      std::vector<dest_t> literals;
       if (block.getNLiterals()) {
         // note: here we have to use md.nLiterals (original number of literal words) rather than md.nLiteralWords == block.getNLiterals()
         // (number of W-words in the EncodedBlock occupied by literals) as we cast literals stored in W-word array
         // to D-word array
-        literals = std::vector<D>{reinterpret_cast<const D*>(block.getLiterals()), reinterpret_cast<const D*>(block.getLiterals()) + md.nLiterals};
+        literals = std::vector<dest_t>{reinterpret_cast<const dest_t*>(block.getLiterals()), reinterpret_cast<const dest_t*>(block.getLiterals()) + md.nLiterals};
       }
       decoder->process(dest, block.getData() + block.getNData(), md.messageLength, literals);
     } else { // data was stored as is
-      std::memcpy(dest, block.payload, md.messageLength * sizeof(D));
+      using destPtr_t = typename std::iterator_traits<D_IT>::pointer;
+      destPtr_t srcBegin = reinterpret_cast<destPtr_t>(block.payload);
+      destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t);
+      std::copy(srcBegin, srcEnd, dest);
+      //std::memcpy(dest, block.payload, md.messageLength * sizeof(dest_t));
     }
   }
 }
@@ -738,7 +763,7 @@ void EncodedBlocks<H, N, W>::encode(const S_IT srcBegin,     // iterator begin o
   mRegistry.nFilledBlocks++;
   using STYP = typename std::iterator_traits<S_IT>::value_type;
   using stream_t = typename o2::rans::Encoder64<STYP>::stream_t;
-  ;
+
   const size_t messageLength = std::distance(srcBegin, srcEnd);
   // cover three cases:
   // * empty source message: no entropy coding
@@ -825,6 +850,7 @@ void EncodedBlocks<H, N, W>::encode(const S_IT srcBegin,     // iterator begin o
     // no dictionary needed
     expandStorage(dataSize);
     *meta = Metadata{messageLength, 0, sizeof(uint64_t), sizeof(stream_t), probabilityBits, opt, 0, 0, 0, dataSize, 0};
+    //FIXME: no we don't need an intermediate vector.
     // provided iterator is not necessarily pointer, need to use intermediate vector!!!
     std::vector<STYP> vtmp(srcBegin, srcEnd);
     bl->storeData(meta->nDataWords, reinterpret_cast<const W*>(vtmp.data()));

DataFormats/Detectors/TPC/include/DataFormatsTPC/CTF.h

@@ -33,6 +33,8 @@ struct CTFHeader : public CompressedClustersCounters {
 /// wrapper for the Entropy-encoded clusters of the TF
 struct CTF : public o2::ctf::EncodedBlocks<CTFHeader, 23, uint32_t> {
 
+  using container_t = o2::ctf::EncodedBlocks<CTFHeader, 23, uint32_t>;
+
   static constexpr size_t N = getNBlocks();
   static constexpr int NBitsQTot = 16;
   static constexpr int NBitsQMax = 10;

Detectors/Base/include/DetectorsBase/CTFCoderBase.h

@@ -59,10 +59,13 @@ class CTFCoderBase
   template <typename S>
   void createCoder(OpType op, const o2::rans::FrequencyTable& freq, uint8_t probabilityBits, int slot)
   {
-    if (op == OpType::Encoder) {
-      mCoders[slot].reset(new o2::rans::LiteralEncoder64<S>(freq, probabilityBits));
-    } else {
-      mCoders[slot].reset(new o2::rans::LiteralDecoder64<S>(freq, probabilityBits));
+    switch (op) {
+      case OpType::Encoder:
+        mCoders[slot].reset(new o2::rans::LiteralEncoder64<S>(freq, probabilityBits));
+        break;
+      case OpType::Decoder:
+        mCoders[slot].reset(new o2::rans::LiteralDecoder64<S>(freq, probabilityBits));
+        break;
     }
   }
 

Detectors/CTF/test/test_ctf_io_tpc.cxx

@@ -42,6 +42,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   {
     CTFCoder coder;
     coder.setCompClusAddresses(c, buff);
+    coder.setCombineColumns(true);
   }
   ccFlat->set(sz, c);
 
@@ -85,6 +86,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   std::vector<o2::ctf::BufferType> vecIO;
   {
     CTFCoder coder;
+    coder.setCombineColumns(true);
     coder.encode(vecIO, c); // compress
   }
   sw.Stop();
@@ -120,6 +122,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
   const auto ctfImage = o2::tpc::CTF::getImage(vecIO.data());
   {
     CTFCoder coder;
+    coder.setCombineColumns(true);
     coder.decode(ctfImage, vecIn); // decompress
   }
   sw.Stop();