Merge pull request #599 from lsst/tickets/DM-11419

DM-11419: Allow larger than 2GB offsets in tables - use std::size_t for offsets…

include/lsst/afw/table/BaseColumnView.h

@@ -41,24 +41,24 @@ class BaseColumnView;
  */
 class BitsColumn final {
 public:
-    typedef std::int64_t IntT;
+    typedef std::size_t SizeT;
 
-    ndarray::Array<IntT, 1, 1> getArray() const { return _array; }
+    ndarray::Array<SizeT, 1, 1> getArray() const { return _array; }
 
-    IntT getBit(Key<Flag> const& key) const;
-    IntT getBit(std::string const& name) const;
+    SizeT getBit(Key<Flag> const& key) const;
+    SizeT getBit(std::string const& name) const;
 
-    IntT getMask(Key<Flag> const& key) const { return IntT(1) << getBit(key); }
-    IntT getMask(std::string const& name) const { return IntT(1) << getBit(name); }
+    SizeT getMask(Key<Flag> const& key) const { return SizeT(1) << getBit(key); }
+    SizeT getMask(std::string const& name) const { return SizeT(1) << getBit(name); }
 
     std::vector<SchemaItem<Flag> > const& getSchemaItems() const { return _items; }
 
 private:
     friend class BaseColumnView;
 
-    explicit BitsColumn(int size);
+    explicit BitsColumn(std::size_t size);
 
-    ndarray::Array<IntT, 1, 1> _array;
+    ndarray::Array<SizeT, 1, 1> _array;
     std::vector<SchemaItem<Flag> > _items;
 };
 
@@ -155,7 +155,7 @@ class BaseColumnView {
     ~BaseColumnView();
 
 protected:
-    BaseColumnView(std::shared_ptr<BaseTable> const& table, int recordCount, void* buf,
+    BaseColumnView(std::shared_ptr<BaseTable> const& table, std::size_t recordCount, void* buf,
                    ndarray::Manager::Ptr const& manager);
 
 private:

include/lsst/afw/table/FieldBase.h

@@ -29,10 +29,10 @@ class TableImpl;
  *
  *  This storage is equivalent to LAPACK 'UPLO=U'.
  */
-inline int indexCovariance(int i, int j) { return (i < j) ? (i + j * (j + 1) / 2) : (j + i * (i + 1) / 2); }
+inline std::size_t indexCovariance(std::size_t i, std::size_t j) { return (i < j) ? (i + j * (j + 1) / 2) : (j + i * (i + 1) / 2); }
 
 /// Defines the packed size of a covariance matrices.
-inline int computeCovariancePackedSize(int size) { return size * (size + 1) / 2; }
+inline std::size_t computeCovariancePackedSize(std::size_t size) { return size * (size + 1) / 2; }
 
 }  // namespace detail
 
@@ -47,7 +47,7 @@ struct FieldBase {
     typedef T Element;                ///< the type of subfields (the same as the type itself for scalars)
 
     /// Return the number of subfield elements (always one for scalars).
-    int getElementCount() const noexcept { return 1; }
+    std::size_t getElementCount() const noexcept { return 1; }
 
     /// Return a string description of the field type.
     static std::string getTypeString();
@@ -56,7 +56,7 @@ struct FieldBase {
     // it's convenient to be able to instantiate both, since the other is used
     // by other specializations.
     FieldBase() = default;
-    FieldBase(int) {
+    FieldBase(std::size_t) {
         throw LSST_EXCEPT(lsst::pex::exceptions::LogicError,
                           "Constructor disabled (this Field type is not sized).");
     }
@@ -119,10 +119,7 @@ struct FieldBase<Array<U> > {
      *
      *  ...even though the third argument to the Field constructor takes a FieldBase, not an int.
      */
-    FieldBase(int size = 0) : _size(size) {
-        if (size < 0)
-            throw LSST_EXCEPT(lsst::pex::exceptions::LogicError,
-                              "A non-negative size must be provided when constructing an array field.");
+    FieldBase(size_t size = 0) : _size(size) {
     }
 
     FieldBase(FieldBase const &) noexcept = default;
@@ -136,11 +133,11 @@ struct FieldBase<Array<U> > {
 
     /// Return the number of subfield elements (equal to the size of the array),
     /// or 0 for a variable-length array.
-    int getElementCount() const noexcept { return _size; }
+    std::size_t getElementCount() const noexcept { return _size; }
 
     /// Return the size of the array (equal to the number of subfield elements),
     /// or 0 for a variable-length array.
-    int getSize() const noexcept { return _size; }
+    std::size_t getSize() const noexcept { return _size; }
 
     /// Return true if the field is variable-length (each record can have a different size array).
     bool isVariableLength() const noexcept { return _size == 0; }
@@ -210,10 +207,10 @@ struct FieldBase<Array<U> > {
             throw LSST_EXCEPT(lsst::pex::exceptions::LengthError,
                               "Incorrect size in array field assignment.");
         }
-        for (int i = 0; i < _size; ++i) p[i] = value[i];
+        for (std::size_t i = 0; i < _size; ++i) p[i] = value[i];
     }
 
-    int _size;
+    std::size_t _size;
 };
 
 /**
@@ -244,7 +241,7 @@ struct FieldBase<std::string> {
      *
      *  ...even though the third argument to the Field constructor takes a FieldBase, not an int.
      */
-    FieldBase(int size = -1);
+    FieldBase(std::size_t size=std::numeric_limits<size_t>::max());
 
     FieldBase(FieldBase const &) noexcept = default;
     FieldBase(FieldBase &&) noexcept = default;
@@ -257,11 +254,11 @@ struct FieldBase<std::string> {
 
     /// @brief Return the number of subfield elements (equal to the size of the string,
     ///        including a null terminator), or 0 for a variable-length string.
-    int getElementCount() const noexcept { return _size; }
+    std::size_t getElementCount() const noexcept { return _size; }
 
     /// @brief Return the maximum length of the string, including a null terminator
     ///        (equal to the number of subfield elements), or 0 for a variable-length string.
-    int getSize() const noexcept { return _size; }
+    std::size_t getSize() const noexcept { return _size; }
 
     /// Return true if the field is variable-length (each record can have a different size array).
     bool isVariableLength() const noexcept { return _size == 0; }
@@ -304,7 +301,7 @@ struct FieldBase<std::string> {
     void setValue(Element *p, ndarray::Manager::Ptr const &, std::string const &value) const;
 
 private:
-    int _size;
+    std::size_t _size;
 };
 }  // namespace table
 }  // namespace afw

include/lsst/afw/table/Flag.h

@@ -35,7 +35,7 @@ struct FieldBase<Flag> {
     typedef std::int64_t Element;  ///< the actual storage type (shared by multiple flag fields)
 
     /// Return the number of subfield elements (always one for scalars).
-    int getElementCount() const { return 1; }
+    std::size_t getElementCount() const { return 1; }
 
     /// Return a string description of the field type.
     static std::string getTypeString() { return "Flag"; }
@@ -44,7 +44,7 @@ struct FieldBase<Flag> {
     // it's convenient to be able to instantiate both, since the other is used
     // by other specializations.
     FieldBase() = default;
-    FieldBase(int) {
+    FieldBase(std::size_t) {
         throw LSST_EXCEPT(lsst::pex::exceptions::LogicError,
                           "Constructor disabled (this Field type is not sized).");
     }
@@ -123,10 +123,10 @@ class Key<Flag> : public KeyBase<Flag>, public FieldBase<Flag> {
     }
 
     /// Return the offset in bytes of the integer element that holds this field's bit.
-    int getOffset() const { return _offset; }
+    std::size_t getOffset() const { return _offset; }
 
     /// The index of this field's bit within the integer it shares with other Flag fields.
-    int getBit() const { return _bit; }
+    std::size_t getBit() const { return _bit; }
 
     /**
      *  Return true if the key was initialized to valid offset.
@@ -136,14 +136,14 @@ class Key<Flag> : public KeyBase<Flag>, public FieldBase<Flag> {
      *
      *  A key that is default constructed will always be invalid.
      */
-    bool isValid() const { return _offset >= 0; }
+    bool isValid() const { return _valid; }
 
     /**
      *  Default construct a field.
      *
      *  The new field will be invalid until a valid Key is assigned to it.
      */
-    Key() : FieldBase<Flag>(), _offset(-1), _bit(0) {}
+    Key() : FieldBase<Flag>(), _offset(0), _bit(0), _valid(false) {}
 
     Key(Key const &) = default;
     Key(Key &&) = default;
@@ -175,10 +175,11 @@ class Key<Flag> : public KeyBase<Flag>, public FieldBase<Flag> {
         }
     }
 
-    explicit Key(int offset, int bit) : _offset(offset), _bit(bit) {}
+    explicit Key(std::size_t offset, std::size_t bit) : _offset(offset), _bit(bit), _valid(true) {}
 
-    int _offset;
-    int _bit;
+    std::size_t _offset;
+    std::size_t _bit;
+    bool _valid;
 };
 }  // namespace table
 }  // namespace afw

include/lsst/afw/table/Key.h

@@ -84,7 +84,7 @@ class Key : public KeyBase<T>, public FieldBase<T> {
     }
 
     /// Return the offset (in bytes) of this field within a record.
-    int getOffset() const noexcept { return _offset; }
+    std::size_t getOffset() const noexcept { return _offset; }
 
     /**
      *  Return true if the key was initialized to valid offset.
@@ -94,14 +94,14 @@ class Key : public KeyBase<T>, public FieldBase<T> {
      *
      *  A key that is default constructed will always be invalid.
      */
-    bool isValid() const noexcept { return _offset >= 0; }
+    bool isValid() const noexcept { return _valid; }
 
     /**
      *  Default construct a field.
      *
      *  The new field will be invalid until a valid Key is assigned to it.
      */
-    Key() noexcept : FieldBase<T>(FieldBase<T>::makeDefault()), _offset(-1) {}
+    Key() noexcept : FieldBase<T>(FieldBase<T>::makeDefault()), _offset(0), _valid(false) {}
 
     Key(Key const&) noexcept = default;
     Key(Key&&) noexcept = default;
@@ -119,10 +119,11 @@ class Key : public KeyBase<T>, public FieldBase<T> {
     friend class detail::Access;
     friend class BaseRecord;
 
-    explicit Key(int offset, FieldBase<T> const& fb = FieldBase<T>()) noexcept
-            : FieldBase<T>(fb), _offset(offset) {}
+    explicit Key(std::size_t offset, FieldBase<T> const& fb = FieldBase<T>()) noexcept
+            : FieldBase<T>(fb), _offset(offset), _valid(true) {}
 
-    int _offset;
+    std::size_t _offset;
+    bool _valid;
 };
 }  // namespace table
 }  // namespace afw

include/lsst/afw/table/KeyBase.h

@@ -32,9 +32,9 @@ class KeyBase<Array<U> > {
 
     void assignVector(BaseRecord& record, std::vector<U> const& values) const;
 
-    Key<U> operator[](int i) const;  ///< Return a subfield key for the i-th element of the array.
+    Key<U> operator[](std::size_t i) const;  ///< Return a subfield key for the i-th element of the array.
 
-    Key<Array<U> > slice(int begin, int end) const;  ///< Return a key for a range of elements
+    Key<Array<U> > slice(std::size_t begin, std::size_t end) const;  ///< Return a key for a range of elements
 };
 }  // namespace table
 }  // namespace afw

include/lsst/afw/table/Schema.h

@@ -146,16 +146,16 @@ class Schema final {
     std::set<std::string> getNames(bool topOnly = false) const;
 
     /// Return the raw size of a record in bytes.
-    int getRecordSize() const { return _impl->getRecordSize(); }
+    std::size_t getRecordSize() const { return _impl->getRecordSize(); }
 
     /// The total number of fields.
-    int getFieldCount() const { return _impl->getFieldCount(); }
+    std::size_t getFieldCount() const { return _impl->getFieldCount(); }
 
     /// The number of Flag fields.
-    int getFlagFieldCount() const { return _impl->getFlagFieldCount(); }
+    std::size_t getFlagFieldCount() const { return _impl->getFlagFieldCount(); }
 
     /// The number of non-Flag fields.
-    int getNonFlagFieldCount() const { return _impl->getNonFlagFieldCount(); }
+    std::size_t getNonFlagFieldCount() const { return _impl->getNonFlagFieldCount(); }
 
     /**
      *  Add a new field to the Schema, and return the associated Key.

include/lsst/afw/table/arrays.h

@@ -72,7 +72,7 @@ class ArrayKey : public FunctorKey<ndarray::Array<T const, 1, 1> >,
      *  @param[in]     size    Number of fields to add.
      */
     static ArrayKey addFields(Schema& schema, std::string const& name, std::string const& doc,
-                              std::string const& unit, int size);
+                              std::string const& unit, size_t size);
 
     /// Default constructor; instance will not be usable unless subsequently assigned to.
     ArrayKey() noexcept : _begin(), _size(0) {}
@@ -96,7 +96,7 @@ class ArrayKey : public FunctorKey<ndarray::Array<T const, 1, 1> >,
      *
      *      ArrayKey<T> k(schema["a"]);
      */
-    ArrayKey(SubSchema const& s);
+    explicit ArrayKey(SubSchema const& s);
 
     ArrayKey(ArrayKey const&) noexcept;
     ArrayKey(ArrayKey&&) noexcept;
@@ -105,7 +105,7 @@ class ArrayKey : public FunctorKey<ndarray::Array<T const, 1, 1> >,
     ~ArrayKey() noexcept override;
 
     /// Return the number of elements in the array.
-    int getSize() const noexcept { return _size; }
+    [[nodiscard]] std::size_t getSize() const noexcept { return _size; }
 
     /// Get an array from the given record
     ndarray::Array<T const, 1, 1> get(BaseRecord const& record) const override;
@@ -128,25 +128,25 @@ class ArrayKey : public FunctorKey<ndarray::Array<T const, 1, 1> >,
     //@}
 
     /// Return a hash of this object.
-    std::size_t hash_value() const noexcept {
+    [[nodiscard]] std::size_t hash_value() const noexcept {
         // Completely arbitrary seed
         return utils::hashCombine(17, _begin, _size);
     }
 
     /// Return True if the FunctorKey contains valid scalar keys.
-    bool isValid() const noexcept { return _begin.isValid(); }
+    [[nodiscard]] bool isValid() const noexcept { return _begin.isValid(); }
 
     /// Return a scalar Key for an element of the array
-    Key<T> operator[](int i) const;
+    Key<T> operator[](std::size_t i) const;
 
     /// Return a FunctorKey corresponding to a range of elements
-    ArrayKey slice(int begin, int end) const;
+    ArrayKey slice(std::size_t begin, std::size_t end) const;
 
 private:
-    ArrayKey(Key<T> const& begin, int size) noexcept : _begin(begin), _size(size) {}
+    ArrayKey(Key<T> const& begin, std::size_t size) noexcept : _begin(begin), _size(size) {}
 
     Key<T> _begin;
-    int _size;
+    std::size_t _size;
 };
 }  // namespace table
 }  // namespace afw

include/lsst/afw/table/detail/Access.h

@@ -32,7 +32,7 @@ class Access final {
 public:
     /// @internal Return a sub-field key corresponding to the nth element.
     template <typename T>
-    static Key<typename Key<T>::Element> extractElement(KeyBase<T> const &kb, int n) {
+    static Key<typename Key<T>::Element> extractElement(KeyBase<T> const &kb, std::size_t n) {
         if (!static_cast<Key<T> const &>(kb).isValid()) {
             throw LSST_EXCEPT(
                     pex::exceptions::LogicError,
@@ -48,7 +48,7 @@ class Access final {
 
     /// @internal Return a sub-field key corresponding to a range
     template <typename T>
-    static Key<Array<T> > extractRange(KeyBase<Array<T> > const &kb, int begin, int end) {
+    static Key<Array<T> > extractRange(KeyBase<Array<T> > const &kb, std::size_t begin, std::size_t end) {
         if (!static_cast<Key<Array<T> > const &>(kb).isValid()) {
             throw LSST_EXCEPT(pex::exceptions::LogicError,
                               (boost::format("Cannot extract subfield key from invalid key of type '%s' ") %
@@ -62,30 +62,30 @@ class Access final {
 
     /// @internal Access to the private Key constructor.
     template <typename T>
-    static Key<T> makeKey(int offset) {
+    static Key<T> makeKey(std::size_t offset) {
         return Key<T>(offset);
     }
 
     /// @internal Access to the private Key constructor.
     template <typename T>
-    static Key<T> makeKey(Field<T> const &field, int offset) {
+    static Key<T> makeKey(Field<T> const &field, std::size_t offset) {
         return Key<T>(offset, field);
     }
 
     /// @internal Access to the private Key constructor.
-    static Key<Flag> makeKey(int offset, int bit) { return Key<Flag>(offset, bit); }
+    static Key<Flag> makeKey(std::size_t offset, std::size_t bit) { return Key<Flag>(offset, bit); }
 
     /// @internal Access to the private Key constructor.
-    static Key<std::string> makeKeyString(int offset, int size) { return Key<std::string>(offset, size); }
+    static Key<std::string> makeKeyString(std::size_t offset, std::size_t size) { return Key<std::string>(offset, size); }
 
     /// @internal Access to the private Key constructor.
     template <typename T>
-    static Key<Array<T>> makeKeyArray(int offset, int size) {
+    static Key<Array<T>> makeKeyArray(std::size_t offset, std::size_t size) {
         return Key<Array<T>>(offset, size);
     }
 
     /// @internal Add some padding to a schema without adding a field.
-    static void padSchema(Schema &schema, int bytes) {
+    static void padSchema(Schema &schema, std::size_t bytes) {
         schema._edit();
         schema._impl->_recordSize += bytes;
     }