Merge the 64-bit changes from casacore

This merges the changes from casacore/casacore#1166 back to here, and changes a few other things to be in line with casacore.
After merging a new version should be released.

dyscostman.cc

@@ -164,7 +164,7 @@ casacore::Bool DyscoStMan::flush(casacore::AipsIO &,
   return false;
 }
 
-void DyscoStMan::create(casacore::uInt nRow) {
+void DyscoStMan::create64(casacore::rownr_t nRow) {
   _nRow = nRow;
   _fStream.reset(new std::fstream(
       fileName().c_str(),
@@ -281,7 +281,7 @@ void DyscoStMan::initializeRowsPerBlock(size_t rowsPerBlock,
   if (writeToHeader) writeHeader();
 }
 
-void DyscoStMan::open(casacore::uInt nRow, casacore::AipsIO &) {
+casacore::rownr_t DyscoStMan::open64(casacore::rownr_t nRow, casacore::AipsIO &) {
   _nRow = nRow;
   _fStream.reset(new std::fstream(fileName().c_str(),
                                   std::ios_base::in | std::ios_base::out));
@@ -302,6 +302,7 @@ void DyscoStMan::open(casacore::uInt nRow, casacore::AipsIO &) {
     _nBlocksInFile = (size_t(size) - _headerSize) / _blockSize;
   else
     _nBlocksInFile = 0;
+  return nRow;
 }
 
 casacore::DataManagerColumn *DyscoStMan::makeScalarColumn(
@@ -344,7 +345,10 @@ casacore::DataManagerColumn *DyscoStMan::makeIndArrColumn(
       "column desc constructor");
 }
 
-void DyscoStMan::resync(casacore::uInt /*nRow*/) {}
+casacore::rownr_t DyscoStMan::resync64(casacore::rownr_t nRow)
+{
+  return nRow;
+}
 
 void DyscoStMan::deleteManager() { unlink(fileName().c_str()); }
 
@@ -377,9 +381,9 @@ void DyscoStMan::prepare() {
 
 void DyscoStMan::reopenRW() {}
 
-void DyscoStMan::addRow(casacore::uInt nrrow) { _nRow += nrrow; }
+void DyscoStMan::addRow64(casacore::rownr_t nrrow) { _nRow += nrrow; }
 
-void DyscoStMan::removeRow(casacore::uInt rowNr) {
+void DyscoStMan::removeRow64(casacore::rownr_t rowNr) {
   if (rowNr != _nRow - 1)
     throw DyscoStManError(
         "Trying to remove a row in the middle of the file: "

dyscostman.h

@@ -51,7 +51,7 @@ class DyscoStMan : public casacore::DataManager {
    * initialized to AF normalization with a truncated Gaussian distribution for
    * the quantization, and a truncation of sigma = 2.5. To change the settings,
    * use one of the Set...Distribution() methods and SetNormalization().
-   * @param databitRate The number of bits per float used for visibilities.
+   * @param dataBitRate The number of bits per float used for visibilities.
    * @param weightBitRate The number of bits per float used for the weight
    * column.
    * @param name Storage manager name.
@@ -299,6 +299,7 @@ class DyscoStMan : public casacore::DataManager {
    * To be called by a column once it determines rowsPerBlock and antennaCount.
    * @param rowsPerBlock Number of measurement set rows in one time block.
    * @param antennaCount Highest antenna index+1 used in a time block.
+   * @param writeToHeader Write the header?
    */
   void initializeRowsPerBlock(size_t rowsPerBlock, size_t antennaCount,
                               bool writeToHeader);
@@ -334,11 +335,11 @@ class DyscoStMan : public casacore::DataManager {
 
   // Let the storage manager create files as needed for a new table.
   // This allows a column with an indirect array to create its file.
-  virtual void create(casacore::uInt nRow) final override;
+  virtual void create64(casacore::rownr_t nRow) final override;
 
   // Open the storage manager file for an existing table.
   // Return the number of rows in the data file.
-  virtual void open(casacore::uInt nRow, casacore::AipsIO &) final override;
+  virtual casacore::rownr_t open64(casacore::rownr_t nRow, casacore::AipsIO &) final override;
 
   // Create a column in the storage manager on behalf of a table column.
   // The caller will NOT delete the newly created object.
@@ -357,7 +358,7 @@ class DyscoStMan : public casacore::DataManager {
       const casacore::String &name, int dataType,
       const casacore::String &dataTypeID) final override;
 
-  virtual void resync(casacore::uInt nRow) final override;
+  virtual casacore::rownr_t resync64(casacore::rownr_t nRow) final override;
 
   virtual void deleteManager() final override;
 
@@ -371,10 +372,10 @@ class DyscoStMan : public casacore::DataManager {
   virtual void reopenRW() final override;
 
   // Add rows to the storage manager.
-  virtual void addRow(casacore::uInt nrrow) final override;
+  virtual void addRow64(casacore::rownr_t nrrow) final override;
 
   // Delete a row from all columns.
-  virtual void removeRow(casacore::uInt rowNr) final override;
+  virtual void removeRow64(casacore::rownr_t rowNr) final override;
 
   // Do the final addition of a column.
   virtual void addColumn(casacore::DataManagerColumn *) final override;

dyscostmancol.h

@@ -4,7 +4,7 @@
 #include "dyscodistribution.h"
 #include "dysconormalization.h"
 
-#include <casacore/tables/DataMan/StManColumn.h>
+#include <casacore/tables/DataMan/StManColumnBase.h>
 
 #include <casa/Arrays/IPosition.h>
 
@@ -19,15 +19,15 @@ class DyscoStMan;
  * Base class for columns of the DyscoStMan.
  * @author André Dysco
  */
-class DyscoStManColumn : public casacore::StManColumn {
+class DyscoStManColumn : public casacore::StManColumnBase {
  public:
   /**
    * Constructor, to be overloaded by subclass.
    * @param parent The parent stman to which this column belongs.
    * @param dtype The column's type as defined by Casacore.
    */
   explicit DyscoStManColumn(DyscoStMan *parent, int dtype)
-      : casacore::StManColumn(dtype),
+      : casacore::StManColumnBase(dtype),
         _offsetInBlock(0),
         _storageManager(parent) {}
 
@@ -75,15 +75,17 @@ class DyscoStManColumn : public casacore::StManColumn {
 
   /**
    * Read a row of compressed data from the stman file.
-   * @param rowIndex The index of the row to read.
+   * @param blockIndex The block index of the row to read.
    * @param dest The destination buffer, should be at least of size Stride().
+   * @param size The nr of bytes to be read.
    */
   void readCompressedData(size_t blockIndex, unsigned char *dest, size_t size);
 
   /**
    * Write a row of compressed data to the stman file.
-   * @param rowIndex The index of the row to write.
+   * @param blockIndex The block index of the row to write.
    * @param data The data buffer containing Stride() bytes.
+   * @param size The nr of bytes to be written.
    */
   void writeCompressedData(size_t blockIndex, const unsigned char *data,
                            size_t size);

stochasticencoder.h

@@ -98,6 +98,7 @@ class StochasticEncoder {
    * Use Decode() on the returned symbol to get
    * the decoded value.
    * @param value Floating point value to be encoded.
+   * @param ditherValue The dithering value to apply.
    */
   symbol_t EncodeWithDithering(ValueType value, unsigned ditherValue) const {
     if (std::isfinite(value)) {

threadeddyscocolumn.cc

@@ -105,21 +105,20 @@ void ThreadedDyscoColumn<DataType>::loadBlock(size_t blockIndex) {
 
 template <typename DataType>
 void ThreadedDyscoColumn<DataType>::getValues(
-    casacore::uInt rowNr, casacore::Array<DataType> *dataPtr) {
+    casacore::rownr_t rowNr, casacore::Array<DataType> *dataArr) {
+  // Make sure array storage is contiguous.
+  casacore::Bool deleteIt;
+  DataType* dataPtr = dataArr->getStorage (deleteIt);
   if (!areOffsetsInitialized()) {
     // Trying to read before first block was written -- return zero
     // TODO if a few rows were written of the first block, those are
     // incorrectly returned. This is a rare case but can be fixed.
-    for (typename casacore::Array<DataType>::contiter i = dataPtr->cbegin();
-         i != dataPtr->cend(); ++i)
-      *i = DataType();
+    *dataPtr = DataType();
   } else {
     size_t blockIndex = getBlockIndex(rowNr);
     if (blockIndex >= nBlocksInFile()) {
       // Trying to read a row that was not stored yet -- return zero
-      for (typename casacore::Array<DataType>::contiter i = dataPtr->cbegin();
-           i != dataPtr->cend(); ++i)
-        *i = DataType();
+      *dataPtr = DataType();
     } else {
       std::unique_lock<std::mutex> lock(_mutex);
       // Wait until the block to be read is not in the write cache
@@ -137,9 +136,10 @@ void ThreadedDyscoColumn<DataType>::getValues(
 
       // The time block encoder is now initialized and contains the unpacked
       // block.
-      _timeBlockBuffer->GetData(getRowWithinBlock(rowNr), dataPtr->data());
+      _timeBlockBuffer->GetData(getRowWithinBlock(rowNr), dataPtr);
     }
   }
+  dataArr->putStorage (dataPtr, deleteIt);
 }
 
 template <typename DataType>
@@ -168,7 +168,10 @@ void ThreadedDyscoColumn<DataType>::storeBlock() {
 
 template <typename DataType>
 void ThreadedDyscoColumn<DataType>::putValues(
-    casacore::uInt rowNr, const casacore::Array<DataType> *dataPtr) {
+    casacore::rownr_t rowNr, const casacore::Array<DataType> *dataArr) {
+  // Make sure array storage is contiguous.
+  casacore::Bool deleteIt;
+  const DataType* dataPtr = dataArr->getStorage (deleteIt);
   if (!areOffsetsInitialized()) {
     // If the manager did not initialize its offsets yet, then it is determined
     // from the first "time block" (a block with the same time, field and spw)
@@ -207,11 +210,12 @@ void ThreadedDyscoColumn<DataType>::putValues(
       // Load new block
       loadBlock(blockIndex);
     }
-    _timeBlockBuffer->SetData(blockRow, ant1, ant2, dataPtr->data());
+    _timeBlockBuffer->SetData(blockRow, ant1, ant2, dataPtr);
   } else {
-    _timeBlockBuffer->SetData(rowNr, ant1, ant2, dataPtr->data());
+    _timeBlockBuffer->SetData(rowNr, ant1, ant2, dataPtr);
   }
   _isCurrentBlockChanged = true;
+  dataArr->freeStorage (dataPtr, deleteIt);
 }
 
 template <typename DataType>

threadeddyscocolumn.h

@@ -24,7 +24,7 @@ namespace dyscostman {
 class DyscoStMan;
 
 /**
- * A column for storing compressed values in a threaded way, tailered for the
+ * A column for storing compressed values in a threaded way, tailored for the
  * data and weight columns that use a threaded approach for encoding.
  * @author André Offringa
  */
@@ -50,8 +50,8 @@ class ThreadedDyscoColumn : public DyscoStManColumn {
   virtual void setShapeColumn(const casacore::IPosition &shape) override;
 
   /** Get the dimensions of the values in a particular row.
-   * @param rownr The row to get the shape for. */
-  virtual casacore::IPosition shape(casacore::uInt /*rownr*/) override {
+   * The rownr parameter is not used as the shape is the same for all rows. */
+  virtual casacore::IPosition shape(casacore::rownr_t /*rownr*/) override {
     return _shape;
   }
 
@@ -61,18 +61,10 @@ class ThreadedDyscoColumn : public DyscoStManColumn {
    * @param rowNr The row number to get the values for.
    * @param dataPtr The array of values, which should be a contiguous array.
    */
-  virtual void getArrayComplexV(
-      casacore::uInt rowNr,
-      casacore::Array<casacore::Complex> *dataPtr) override {
-    // Note that this method is specialized for std::complex<float> -- the
-    // generic method won't do anything
-    return DyscoStManColumn::getArrayComplexV(rowNr, dataPtr);
-  }
-  virtual void getArrayfloatV(casacore::uInt rowNr,
-                              casacore::Array<float> *dataPtr) override {
-    // Note that this method is specialized for float -- the generic method
-    // won't do anything
-    return DyscoStManColumn::getArrayfloatV(rowNr, dataPtr);
+  virtual void getArrayV(
+      casacore::rownr_t rowNr,
+      casacore::ArrayBase &dataPtr) override {
+      return DyscoStManColumn::getArrayV(rowNr, dataPtr);
   }
 
   /**
@@ -82,18 +74,10 @@ class ThreadedDyscoColumn : public DyscoStManColumn {
    * @param rowNr The row number to write the values to.
    * @param dataPtr The data pointer, which should be a contiguous array.
    */
-  virtual void putArrayComplexV(
-      casacore::uInt rowNr,
-      const casacore::Array<casacore::Complex> *dataPtr) override {
-    // Note that this method is specialized for std::complex<float> -- the
-    // generic method won't do anything
-    return DyscoStManColumn::putArrayComplexV(rowNr, dataPtr);
-  }
-  virtual void putArrayfloatV(casacore::uInt rowNr,
-                              const casacore::Array<float> *dataPtr) override {
-    // Note that this method is specialized for float -- the generic method
-    // won't do anything
-    return DyscoStManColumn::putArrayfloatV(rowNr, dataPtr);
+  virtual void putArrayV(
+      casacore::rownr_t rowNr,
+      const casacore::ArrayBase &dataPtr) override {
+    return DyscoStManColumn::putArrayV(rowNr, dataPtr);
   }
 
   virtual void Prepare(DyscoDistribution distribution,
@@ -190,8 +174,8 @@ class ThreadedDyscoColumn : public DyscoStManColumn {
 
   typedef std::map<size_t, CacheItem *> cache_t;
 
-  void getValues(casacore::uInt rowNr, casacore::Array<data_t> *dataPtr);
-  void putValues(casacore::uInt rowNr, const casacore::Array<data_t> *dataPtr);
+  void getValues(casacore::rownr_t rowNr, casacore::Array<data_t> *dataPtr);
+  void putValues(casacore::rownr_t rowNr, const casacore::Array<data_t> *dataPtr);
 
   void stopThreads();
   void encodeAndWrite(size_t blockIndex, const CacheItem &item,
@@ -228,24 +212,24 @@ class ThreadedDyscoColumn : public DyscoStManColumn {
 };
 
 template <>
-inline void ThreadedDyscoColumn<std::complex<float>>::getArrayComplexV(
-    casacore::uInt rowNr, casacore::Array<casacore::Complex> *dataPtr) {
-  getValues(rowNr, dataPtr);
+inline void ThreadedDyscoColumn<std::complex<float>>::getArrayV(
+    casacore::rownr_t rowNr, casacore::ArrayBase &dataPtr) {
+  getValues(rowNr, static_cast<casacore::Array<std::complex<float>>*>(&dataPtr));
 }
 template <>
-inline void ThreadedDyscoColumn<std::complex<float>>::putArrayComplexV(
-    casacore::uInt rowNr, const casacore::Array<casacore::Complex> *dataPtr) {
-  putValues(rowNr, dataPtr);
+inline void ThreadedDyscoColumn<std::complex<float>>::putArrayV(
+    casacore::rownr_t rowNr, const casacore::ArrayBase &dataPtr) {
+  putValues(rowNr, static_cast<const casacore::Array<std::complex<float>>*>(&dataPtr));
 }
 template <>
-inline void ThreadedDyscoColumn<float>::getArrayfloatV(
-    casacore::uInt rowNr, casacore::Array<float> *dataPtr) {
-  getValues(rowNr, dataPtr);
+inline void ThreadedDyscoColumn<float>::getArrayV(
+    casacore::rownr_t rowNr, casacore::ArrayBase &dataPtr) {
+  getValues(rowNr, static_cast<casacore::Array<float>*>(&dataPtr));
 }
 template <>
-inline void ThreadedDyscoColumn<float>::putArrayfloatV(
-    casacore::uInt rowNr, const casacore::Array<float> *dataPtr) {
-  putValues(rowNr, dataPtr);
+inline void ThreadedDyscoColumn<float>::putArrayV(
+    casacore::rownr_t rowNr, const casacore::ArrayBase &dataPtr) {
+  putValues(rowNr, static_cast<const casacore::Array<float>*>(&dataPtr));
 }
 
 extern template class ThreadedDyscoColumn<std::complex<float>>;