Added VARBINARY support for ARIES

src/ee/common/CompactingStringPool.cpp

@@ -0,0 +1,54 @@
+/* This file is part of VoltDB.
+ * Copyright (C) 2008-2011 VoltDB Inc.
+ *
+ * VoltDB is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * VoltDB is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "CompactingStringPool.h"
+
+#include "StringRef.h"
+
+using namespace voltdb;
+using namespace std;
+
+CompactingStringPool::CompactingStringPool(int32_t elementSize,
+                                           int32_t elementsPerBuf) :
+    m_pool(elementSize, elementsPerBuf)
+{
+}
+
+void*
+CompactingStringPool::malloc()
+{
+    return m_pool.malloc();
+}
+
+void
+CompactingStringPool::free(void* element)
+{
+    bool mutated = m_pool.free(element);
+    if (mutated)
+    {
+        // use the backpointer to the StringRef object in the moved
+        // data to update that object with the new string location
+        StringRef* back_ptr = *reinterpret_cast<StringRef**>(element);
+        back_ptr->updateStringLocation(element);
+    }
+}
+
+size_t
+CompactingStringPool::getBytesAllocated() const
+{
+    return m_pool.getBytesAllocated();
+}

src/ee/common/CompactingStringPool.h

@@ -0,0 +1,42 @@
+/* This file is part of VoltDB.
+ * Copyright (C) 2008-2011 VoltDB Inc.
+ *
+ * VoltDB is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * VoltDB is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _EE_COMMON_COMPACTINGSTRINGPOOL_H_
+#define _EE_COMMON_COMPACTINGSTRINGPOOL_H_
+
+#include "structures/CompactingPool.h"
+
+#include <cstdlib>
+
+namespace voltdb
+{
+    class CompactingStringPool
+    {
+    public:
+        CompactingStringPool(int32_t elementSize, int32_t elementsPerBuf);
+
+        void* malloc();
+        void free(void* element);
+        size_t getBytesAllocated() const;
+
+    private:
+        CompactingPool m_pool;
+    };
+}
+
+
+#endif // _EE_COMMON_COMPACTINGSTRINGPOOL_H_

src/ee/common/CompactingStringStorage.cpp

@@ -0,0 +1,164 @@
+/* This file is part of VoltDB.
+ * Copyright (C) 2008-2011 VoltDB Inc.
+ *
+ * VoltDB is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * VoltDB is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "common/CompactingStringStorage.h"
+
+#include "common/FatalException.hpp"
+#include <boost/unordered_map.hpp>
+#include <iostream>
+
+using namespace voltdb;
+using namespace std;
+using boost::shared_ptr;
+
+namespace
+{
+    size_t getAllocationSizeForObject(size_t length)
+    {
+        if (length <= 2) {
+            return 2;
+        } else if (length <= 4) {
+            return 4;
+        } else if (length <= 4 + 2) {
+            return 4 + 2;
+        } else if (length <= 8) {
+            return 8;
+        } else if (length <= 8 + 4) {
+            return 8 + 4;
+        } else if (length <= 16) {
+            return 16;
+        } else if (length <= 16 + 8) {
+            return 16 + 8;
+        } else if (length <= 32) {
+            return 32;
+        } else if (length <= 32 + 16) {
+            return 32 + 16;
+        } else if (length <= 64) {
+            return 64;
+        } else if (length <= 64 + 32) {
+            return 64 + 32;
+        } else if (length <= 128) {
+            return 128;
+        } else if (length < 128 + 64) {
+            return 128 + 64;
+        } else if (length <= 256) {
+            return 256;
+        } else if (length <= 256 + 128) {
+            return 256 + 128;
+        } else if (length <= 512) {
+            return 512;
+        } else if (length <= 512 + 256) {
+            return 512 + 256;
+        } else if (length <= 1024) {
+            return 1024;
+        } else if (length <= 1024 + 512) {
+            return 1024 + 512;
+        } else if (length <= 2048) {
+            return 2048;
+        } else if (length <= 2048 + 1024) {
+            return 2048 + 1024;
+        } else if (length <= 4096) {
+            return 4096;
+        } else if (length < 4096 + 2048) {
+            return 4096 + 2048;
+        } else if (length <= 8192) {
+            return 8192;
+        } else if (length < 8192 + 4096) {
+            return 8192 + 4096;
+        } else if (length <= 16384) {
+            return 16384;
+        } else if (length <= 16384 + 8192) {
+            return 16384 + 8192;
+        } else if (length <= 32768) {
+            return 32768;
+        } else if (length <= 32768 + 16384) {
+            return 32768 + 16384;
+        } else if (length <= 65536) {
+            return 65536;
+        } else if (length <= 65536 + 32768) {
+            return 65536 + 32768;
+        } else if (length <= 131072) {
+            return 131072;
+        } else if (length <= 131072 + 65536) {
+            return 131072 + 65536;
+        } else if (length <= 262144) {
+            return 262144;
+        } else if (length <= 262144 + 131072) {
+            return 262144 + 131072;
+        } else if (length <= 524288) {
+            return 524288;
+        } else if (length <= 524288 + 262144) {
+            return 524288 + 262144;
+            //Need space for a length prefix and a backpointer
+        } else if (length <= 1048576 + sizeof(int32_t) + sizeof(void*)) {
+            return 1048576 + sizeof(int32_t) + sizeof(void*);
+        } else {
+            throwFatalException("Attempted to allocate an object then the 1 meg limit. Requested size was %Zu", length);
+        }
+        // NOT REACHED
+        return length + 4;
+    }
+}
+
+typedef boost::shared_ptr<CompactingStringPool> PoolPtrType;
+typedef boost::unordered_map<size_t, PoolPtrType> MapType;
+
+CompactingStringStorage::CompactingStringStorage()
+{
+}
+
+CompactingStringStorage::~CompactingStringStorage()
+{
+}
+
+PoolPtrType
+CompactingStringStorage::get(size_t size)
+{
+    size = getAllocationSizeForObject(size);
+    return getExact(size);
+}
+
+PoolPtrType
+CompactingStringStorage::getExact(size_t size)
+{
+    MapType::iterator iter = m_poolMap.find(size);
+    int32_t ssize = static_cast<int32_t>(size);
+    PoolPtrType pool;
+    if (iter == m_poolMap.end()) {
+        // compute num_elements to be closest multiple
+        // leading to a 2Meg buffer
+        int32_t num_elements = (2 * 1024 * 1024 / ssize) + 1;
+        pool = PoolPtrType(new CompactingStringPool(ssize, num_elements));
+        m_poolMap.insert(pair<size_t, PoolPtrType>(size, pool));
+    }
+    else
+    {
+        pool = iter->second;
+    }
+    return pool;
+}
+
+size_t CompactingStringStorage::getPoolAllocationSize()
+{
+    size_t total = 0;
+    for (MapType::iterator iter = m_poolMap.begin();
+         iter != m_poolMap.end();
+         ++iter)
+    {
+        total += iter->second->getBytesAllocated();
+    }
+    return total;
+}

src/ee/common/CompactingStringStorage.h

@@ -0,0 +1,44 @@
+/* This file is part of VoltDB.
+ * Copyright (C) 2008-2011 VoltDB Inc.
+ *
+ * VoltDB is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * VoltDB is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _EE_COMMON_COMPACTINGSTRINGSTORAGE_H_
+#define _EE_COMMON_COMPACTINGSTRINGSTORAGE_H_
+
+#include "CompactingStringPool.h"
+#include <boost/shared_ptr.hpp>
+#include <boost/unordered_map.hpp>
+
+namespace voltdb {
+
+    class CompactingStringStorage {
+    public:
+        CompactingStringStorage();
+        ~CompactingStringStorage();
+
+        boost::shared_ptr<CompactingStringPool> get(size_t size);
+
+        boost::shared_ptr<CompactingStringPool> getExact(size_t size);
+
+        std::size_t getPoolAllocationSize();
+
+    private:
+        boost::unordered_map<size_t,
+                             boost::shared_ptr<CompactingStringPool> > m_poolMap;
+    };
+}
+
+#endif /* COMPACTINGSTRINGSTORAGE_H_ */

src/ee/common/DefaultTupleSerializer.cpp

@@ -1,5 +1,5 @@
 /* This file is part of VoltDB.
- * Copyright (C) 2008-2010 VoltDB Inc.
+ * Copyright (C) 2008-2011 VoltDB Inc.
  *
  * VoltDB is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -36,6 +36,8 @@ int DefaultTupleSerializer::getMaxSerializedTupleSize(const TupleSchema *schema)
         if (!schema->columnIsInlined(ii)) {
             size -= sizeof(void*);
             size += 4 + schema->columnLength(ii);
+        } else if ((schema->columnType(ii) == VALUE_TYPE_VARCHAR) || (schema->columnType(ii) == VALUE_TYPE_VARBINARY)) {
+            size += 3;//Serialization always uses a 4-byte length prefix
         }
     }
     return static_cast<int>(size);

src/ee/common/DefaultTupleSerializer.h

@@ -1,5 +1,5 @@
 /* This file is part of VoltDB.
- * Copyright (C) 2008-2010 VoltDB Inc.
+ * Copyright (C) 2008-2011 VoltDB Inc.
  *
  * VoltDB is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

src/ee/common/ExportSerializeIo.h

@@ -1,5 +1,5 @@
 /* This file is part of VoltDB.
- * Copyright (C) 2008-2010 VoltDB Inc.
+ * Copyright (C) 2008-2011 VoltDB Inc.
  *
  * This file contains original code and/or modifications of original code.
  * Any modifications made by VoltDB Inc. are licensed under the following
@@ -46,6 +46,12 @@
 #ifndef NATIVEEXPORTSERIALIZEIO_H
 #define NATIVEEXPORTSERIALIZEIO_H
 
+#include <cassert>
+#include <cstring>
+#include <limits>
+#include <stdint.h>
+#include <string>
+
 namespace voltdb {
 
 /*