updated Geners interface to 1.3.0

Alignment/Geners/interface/AbsArchive.hh

@@ -16,9 +16,11 @@ namespace gs {
 gs::AbsArchive& operator<<(gs::AbsArchive& ar, const gs::AbsRecord& record);
 
 namespace gs {
+    //
     // If you need to retrieve the items, use the interface provided
     // by the "Reference" class. Public interface of this class only
-    // allows to examine the item metadata.
+    // allows to examine the item metadata and to copy items as BLOBs.
+    // 
     class AbsArchive
     {
     public:
@@ -63,14 +65,33 @@ namespace gs {
                                 const SearchSpecifier& categoryPattern,
                                 std::vector<unsigned long long>* found) const=0;
 
-        // Fetch metadata for the item with given id
+        // Fetch metadata for the item with given id. NULL pointer is
+        // returned if there is no item in the archive with the given id
+        // (and there is an automatic cast from CPP11_shared_ptr to bool).
         virtual CPP11_shared_ptr<const CatalogEntry> 
         catalogEntry(unsigned long long id) = 0;
 
         // Dump everything to storage (if the archive is open for writing
         // and if this makes sense for the archive)
         virtual void flush() = 0;
 
+        // Copy an item from this archive to a destination archive.
+        // This archive and destination archive must be distinct.
+        // Note that, while it is always possible to copy an item
+        // with correct id, a standalone operation like this is
+        // not necessarily going to make sense because the item
+        // could be just a part of some distributed object.
+        //
+        // If "newName" and/or "newCategory" arguments are left at their
+        // default value of 0, the existing name and/or category are used.
+        //
+        // This method returns the id of the copy in the destination
+        // archive.
+        unsigned long long copyItem(unsigned long long id,
+                                    AbsArchive* destination,
+                                    const char* newName = 0,
+                                    const char* newCategory = 0);
+
         // The id and the length of the last item written
         // (the results make sense only for the archives
         //  that have been open for writing)
@@ -105,7 +126,18 @@ namespace gs {
         // Position the input stream for reading the item with given id.
         // The reading must follow immediately, any other operaction on
         // the archive can invalidate the result.
-        virtual std::istream& inputStream(unsigned long long id) = 0;
+        //
+        // The "sz" argument could be NULL in which case it should be
+        // ignored. If it is not NULL, the *sz value on completion
+        // should be set as follows:
+        //
+        // *sz = -1  -- if the uncompressed object size in the stream
+        //              is the same as in the catalog
+        // *sz >= 0  -- *sz is the object size after decompression
+        //              (usually not the same as in the catalog)
+        //
+        virtual std::istream& inputStream(unsigned long long id,
+                                          long long* sz) = 0;
 
         // Get the stream for writing the next object
         virtual std::ostream& outputStream() = 0;

Alignment/Geners/interface/BinaryArchiveBase.hh

@@ -201,7 +201,7 @@ namespace gs {
                                                unsigned* compressionCode,
                                                unsigned long long* length) = 0;
 
-        std::istream& inputStream(unsigned long long id);
+        std::istream& inputStream(unsigned long long id, long long* sz);
         std::ostream& outputStream();
         std::ostream& compressedStream(std::ostream& uncompressed);
         unsigned flushCompressedRecord(std::ostream& compressed);

Alignment/Geners/interface/Record.hh

@@ -11,12 +11,63 @@ namespace gs {
         return ArchiveRecord<T>(object, name, category);
     }
 
+#ifndef SWIG
+    template <typename T>
+    inline ArchiveRecord<T> Record(const T& object, const std::string& name,
+                                   const char* category)
+    {
+        return ArchiveRecord<T>(object, name.c_str(), category);
+    }
+
+    template <typename T>
+    inline ArchiveRecord<T> Record(const T& object, const char* name,
+                                   const std::string& category)
+    {
+        return ArchiveRecord<T>(object, name, category.c_str());
+    }
+
+    template <typename T>
+    inline ArchiveRecord<T> Record(const T& object, const std::string& name,
+                                   const std::string& category)
+    {
+        return ArchiveRecord<T>(object, name.c_str(), category.c_str());
+    }
+#endif
+
+    //
+    // ValueRecord makes a copy of the object and stores it internally
+    //
     template <typename T>
     inline ArchiveValueRecord<T> ValueRecord(const T& object, const char* name,
                                              const char* category)
     {
         return ArchiveValueRecord<T>(object, name, category);
     }
+
+#ifndef SWIG
+    template <typename T>
+    inline ArchiveValueRecord<T> ValueRecord(const T& object,
+                                             const std::string& name,
+                                             const char* category)
+    {
+        return ArchiveValueRecord<T>(object, name.c_str(), category);
+    }
+
+    template <typename T>
+    inline ArchiveValueRecord<T> ValueRecord(const T& object, const char* name,
+                                             const std::string& category)
+    {
+        return ArchiveValueRecord<T>(object, name, category.c_str());
+    }
+
+    template <typename T>
+    inline ArchiveValueRecord<T> ValueRecord(const T& object,
+                                             const std::string& name,
+                                             const std::string& category)
+    {
+        return ArchiveValueRecord<T>(object, name.c_str(), category.c_str());
+    }
+#endif
 }
 
 #endif // GENERS_RECORD_HH_

Alignment/Geners/interface/Reference.hh

@@ -17,6 +17,23 @@ namespace gs {
             :  AbsReference(ar, ClassId::makeId<T>(), "gs::Single",
                             name, category) {}
 
+#ifndef SWIG
+        inline Reference(AbsArchive& ar, const std::string& name,
+                         const char* category)
+            :  AbsReference(ar, ClassId::makeId<T>(), "gs::Single",
+                            name.c_str(), category) {}
+
+        inline Reference(AbsArchive& ar, const char* name,
+                         const std::string& category)
+            :  AbsReference(ar, ClassId::makeId<T>(), "gs::Single",
+                            name, category.c_str()) {}
+
+        inline Reference(AbsArchive& ar, const std::string& name,
+                         const std::string& category)
+            :  AbsReference(ar, ClassId::makeId<T>(), "gs::Single",
+                            name.c_str(), category.c_str()) {}
+#endif
+
         inline Reference(AbsArchive& ar, const SearchSpecifier& namePattern,
                          const SearchSpecifier& categoryPattern)
             :  AbsReference(ar, ClassId::makeId<T>(), "gs::Single",

Alignment/Geners/interface/StringArchive.hh

@@ -61,7 +61,7 @@ namespace gs {
 
     private:
         void search(AbsReference& reference);
-        std::istream& inputStream(unsigned long long id);
+        std::istream& inputStream(unsigned long long id, long long* sz);
 
         inline std::ostream& outputStream()
         {

Alignment/Geners/interface/stringArchiveIO.hh

@@ -0,0 +1,46 @@
+#ifndef GENERS_STRINGARCHIVEIO_HH_
+#define GENERS_STRINGARCHIVEIO_HH_
+
+#include "Alignment/Geners/interface/StringArchive.hh"
+
+namespace gs {
+    // The following function returns "true" on success,
+    // "false" on failure.
+    bool writeStringArchive(const StringArchive& ar, const char* filename);
+
+    // The following function either succeeds or throws
+    // an exception. The archive is created on the heap
+    // and eventually has to be deleted by the user.
+    StringArchive* readStringArchive(const char* filename);
+
+    // Similar operations with compression
+    bool writeCompressedStringArchive(
+        const StringArchive& ar, const char* filename,
+        unsigned compressionMode = 1U, int compressionLevel = -1,
+        unsigned minSizeToCompress = 1024U, unsigned bufSize = 1048576U);
+
+    StringArchive* readCompressedStringArchive(const char* filename);
+
+    // The following function will write a compressed string archive
+    // using default compression parameters if the file name has the
+    // given suffix, otherwise it will write an uncompressed archive.
+    // If the suffix is not provided (i.e., default value of 0 is used),
+    // ".gssaz" will be assumed.
+    bool writeCompressedStringArchiveExt(const StringArchive& ar,
+                                         const char* filename,
+                                         const char* suffix = 0);
+
+    // The following function will attempt to read a compressed string
+    // archive if the file name has the given suffix, otherwise it will
+    // attempt to read an uncompressed archive. If the suffix is not
+    // provided (i.e., default value of 0 is used), ".gssaz" will be assumed.
+    StringArchive* readCompressedStringArchiveExt(const char* filename,
+                                                  const char* suffix = 0);
+
+    // This function will extract one string archive from another
+    StringArchive* loadStringArchiveFromArchive(AbsArchive& arch,
+                                                unsigned long long id);
+}
+
+#endif // GENERS_STRINGARCHIVEIO_HH_
+

Alignment/Geners/src/AbsArchive.cc

@@ -4,6 +4,52 @@
 #include "Alignment/Geners/interface/AbsArchive.hh"
 #include "Alignment/Geners/interface/IOException.hh"
 
+#define GS_STREAM_COPY_BUFFER_SIZE 65536
+
+static void archive_stream_copy(std::istream &in, std::size_t count,
+                                std::ostream &out)
+{
+    if (count)
+    {
+        const std::size_t bufsize = GS_STREAM_COPY_BUFFER_SIZE;
+        char buffer[bufsize];
+
+        bool in_fail = in.fail();
+        bool out_fail = out.fail();
+        while (count > bufsize && !in_fail && !out_fail)
+        {
+            in.read(buffer, bufsize);
+            in_fail = in.fail();
+            if (!in_fail)
+            {
+                out.write(buffer, bufsize);
+                out_fail = out.fail();
+            }
+            count -= bufsize;
+        }
+        if (!in_fail && !out_fail)
+        {
+            in.read(buffer, count);
+            if (!in.fail())
+                out.write(buffer, count);
+        }
+    }
+}
+
+namespace {
+    class NotWritableRecord : public gs::AbsRecord
+    {
+    public:
+        inline NotWritableRecord(const gs::ClassId& classId,
+                                 const char* ioPrototype,
+                                 const char* name, const char* category)
+            : gs::AbsRecord(classId, ioPrototype, name, category) {}
+    private:
+        NotWritableRecord();
+        inline bool writeData(std::ostream&) const {return false;}
+    };
+}
+
 namespace gs {
     AbsArchive::AbsArchive(const char* name)
         : name_(name ? name : ""),
@@ -17,6 +63,81 @@ namespace gs {
     {
         r.addItemId(id);
     }
+
+    unsigned long long AbsArchive::copyItem(const unsigned long long id,
+                                            AbsArchive* destination,
+                                            const char* newName,
+                                            const char* newCategory)
+    {
+        if (!isReadable())
+            throw gs::IOInvalidArgument("In gs::AbsArchive::copyItem: "
+                                        "origin archive is not readable");
+        assert(destination);
+        if (this == destination)
+            throw gs::IOInvalidArgument("In gs::AbsArchive::copyItem: "
+                         "origin and destination archives are the same");
+        AbsArchive& ar(*destination);
+        if (!ar.isWritable())
+            throw gs::IOInvalidArgument("In gs::AbsArchive::copyItem: "
+                                        "destination archive is not writable");
+        CPP11_shared_ptr<const CatalogEntry> entry(catalogEntry(id));
+        if (!entry)
+            throw gs::IOInvalidArgument("In gs::AbsArchive::copyItem: no item "
+                                        "in the archive with the given id");
+
+        // Position the input stream
+        long long sz = 0;
+        std::istream& is = inputStream(id, &sz);
+
+        // The following code is similar to the one in the "operator<<"
+        // below w.r.t. operations with the output archive
+        std::ostream& os = ar.outputStream();
+        std::streampos base = os.tellp();
+        std::ostream& compressed = ar.compressedStream(os);
+
+        // Transfer the data between the streams
+        unsigned long long len;
+        if (sz >= 0)
+            len = sz;
+        else
+            len = entry->itemLength();
+        archive_stream_copy(is, len, compressed);
+        if (is.fail())
+            throw IOReadFailure("In gs::AbsArchive::copyItem: "
+                                "input stream failure");
+        if (compressed.fail())
+            throw IOWriteFailure("In gs::AbsArchive::copyItem: "
+                                 "output stream failure");
+        const unsigned compressCode = ar.flushCompressedRecord(compressed);
+        if (os.fail())
+            throw IOWriteFailure("In gs::AbsArchive::copyItem: "
+                                 "failed to transfer compressed data");
+
+        // Figure out the record length. Naturally, can't have negative length.
+        std::streamoff off = os.tellp() - base;
+        const long long delta = off;
+        assert(delta >= 0LL);
+
+        // We need to create a record out of the catalog entry we have found
+        const char* name = newName;
+        if (!name)
+            name = entry->name().c_str();
+        const char* category = newCategory;
+        if (!category)
+            category = entry->category().c_str();
+        NotWritableRecord record(entry->type(), entry->ioPrototype().c_str(),
+                                 name, category);
+
+        // Add the record to the catalog
+        const unsigned long long id2 = ar.addToCatalog(
+            record, compressCode, delta);
+        if (id == 0ULL)
+            throw IOWriteFailure("In gs::AbsArchive::copyItem: "
+                                 "failed to add catalog entry");
+        ar.lastItemId_ = id2;
+        ar.lastItemLength_ = delta;
+        return id2;
+    }
 }
 
 static std::string local_error_message(gs::AbsArchive& ar,
@@ -60,7 +181,7 @@ gs::AbsArchive& operator<<(gs::AbsArchive& ar, const gs::AbsRecord& record)
 
     // Figure out the record length. Naturally, can't have negative length.
     std::streamoff off = os.tellp() - base;
-    long long delta = off;
+    const long long delta = off;
     assert(delta >= 0LL);
 
     // Add the record to the catalog

Alignment/Geners/src/AbsReference.cc

@@ -17,7 +17,7 @@ namespace gs {
     std::istream& AbsReference::positionInputStream(
         const unsigned long long id) const
     {
-        return archive_.inputStream(id);
+        return archive_.inputStream(id, 0);
     }
 
     void AbsReference::initialize() const