Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Faster implementation of BufferedInputStream::readVInt32()

  • Loading branch information...
commit ad6cd5886503374a8be22865633ffe69bf185bc2 1 parent 0828790
@lalinsky authored
View
1  CMakeLists.txt
@@ -91,6 +91,7 @@ set(tests_SOURCES
src/index/segment_enum_test.cpp
src/index/segment_merger_test.cpp
src/index/segment_merge_policy_test.cpp
+ src/store/buffered_input_stream_test.cpp
src/store/input_stream_test.cpp
src/store/output_stream_test.cpp
src/store/fs_output_stream_test.cpp
View
31 LICENSE
@@ -18,3 +18,34 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+
+Additional copyright for file src/util/vint.h:
+
+Copyright 2008 Google Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
View
23 src/store/buffered_input_stream.cpp
@@ -1,6 +1,7 @@
// Copyright (C) 2011 Lukas Lalinsky
// Distributed under the MIT license, see the LICENSE file for details.
+#include "util/vint.h"
#include "buffered_input_stream.h"
using namespace Acoustid;
@@ -41,18 +42,20 @@ uint32_t BufferedInputStream::readVInt32()
if (m_position >= m_length) {
refill();
}
- uint8_t b = m_buffer[m_position++];
- uint32_t i = b & 0x7f;
- int shift = 7;
- while (b & 0x80) {
- if (m_position >= m_length) {
- refill();
+ if (m_length - m_position >= kMaxVInt32Bytes) {
+ // We have enough data in the buffer for any vint32, so we can use an
+ // optimized function for reading from memory array.
+ uint32_t result;
+ ssize_t size = readVInt32FromArray(&m_buffer[m_position], &result);
+ if (size == -1) {
+ throw IOException("can't read vint32");
}
- b = m_buffer[m_position++];
- i |= (b & 0x7f) << shift;
- shift += 7;
+ m_position += size;
+ return result;
}
- return i;
+ // We will probably need to refill the buffer in the middle, use the
+ // generic implementation and let readByte() handle that.
+ return InputStream::readVInt32();
}
void BufferedInputStream::refill()
View
76 src/store/buffered_input_stream_test.cpp
@@ -0,0 +1,76 @@
+// Copyright (C) 2011 Lukas Lalinsky
+// Distributed under the MIT license, see the LICENSE file for details.
+
+#include <gtest/gtest.h>
+#include "buffered_input_stream.h"
+
+using namespace Acoustid;
+
+class SimpleBufferedInputStream : public BufferedInputStream
+{
+public:
+ SimpleBufferedInputStream(uint8_t *data) : m_data(data) { }
+
+protected:
+ size_t read(uint8_t *data, size_t offset, size_t length)
+ {
+ memmove(data, m_data + offset, length);
+ }
+
+private:
+ uint8_t *m_data;
+};
+
+TEST(BufferedInputStream, ReadByte)
+{
+ uint8_t data[] = { 0, 0xff, 0x01 };
+ SimpleBufferedInputStream inputStream(data);
+ ASSERT_EQ(0x00, inputStream.readByte());
+ ASSERT_EQ(0xff, inputStream.readByte());
+ ASSERT_EQ(0x01, inputStream.readByte());
+}
+
+TEST(BufferedInputStreamTest, ReadInt16)
+{
+ uint8_t data[] = { 0, 0, 0xff, 0xff, 0x01, 0x02 };
+ SimpleBufferedInputStream inputStream(data);
+ ASSERT_EQ(0x0000, inputStream.readInt16());
+ ASSERT_EQ(0xffff, inputStream.readInt16());
+ ASSERT_EQ(0x0102, inputStream.readInt16());
+}
+
+TEST(BufferedInputStreamTest, ReadInt32)
+{
+ uint8_t data[] = { 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0x01, 0x02, 0x03, 0x04 };
+ SimpleBufferedInputStream inputStream(data);
+ ASSERT_EQ(0x00000000, inputStream.readInt32());
+ ASSERT_EQ(0xffffffff, inputStream.readInt32());
+ ASSERT_EQ(0x01020304, inputStream.readInt32());
+}
+
+TEST(BufferedInputStreamTest, ReadVInt32)
+{
+ uint8_t data[] = {
+ 1,
+ 0x80 | 1, 2,
+ 0x80 | 1, 0x80 | 2, 3,
+ 0x80 | 1, 0x80 | 2, 0x80 | 3, 4,
+ 0x80 | 1, 0x80 | 2, 0x80 | 3, 0x80 | 4, 5,
+ };
+ SimpleBufferedInputStream inputStream(data);
+ ASSERT_EQ(1, inputStream.readVInt32());
+ ASSERT_EQ((2 << 7) | 1, inputStream.readVInt32());
+ ASSERT_EQ((3 << 14) | (2 << 7) | 1, inputStream.readVInt32());
+ ASSERT_EQ((4 << 21) | (3 << 14) | (2 << 7) | 1, inputStream.readVInt32());
+ ASSERT_EQ((5 << 28) | (4 << 21) | (3 << 14) | (2 << 7) | 1, inputStream.readVInt32());
+}
+
+TEST(BufferedInputStreamTest, ReadString)
+{
+ uint8_t data[] = {
+ 4, 't', 'e', 's', 't'
+ };
+ SimpleBufferedInputStream inputStream(data);
+ ASSERT_EQ(std::string("test"), inputStream.readString().toStdString());
+}
+
View
22 src/store/memory_input_stream.cpp
@@ -5,6 +5,7 @@
#include <QFile>
#include <sys/mman.h>
#include "common.h"
+#include "util/vint.h"
#include "memory_input_stream.h"
using namespace Acoustid;
@@ -35,14 +36,19 @@ uint8_t MemoryInputStream::readByte()
uint32_t MemoryInputStream::readVInt32()
{
- uint8_t b = m_addr[m_position++];
- uint32_t i = b & 0x7f;
- int shift = 7;
- while (b & 0x80) {
- b = m_addr[m_position++];
- i |= (b & 0x7f) << shift;
- shift += 7;
+ if (m_length - m_position >= kMaxVInt32Bytes) {
+ // We have enough data in the buffer for any vint32, so we can use an
+ // optimized function for reading from memory array.
+ uint32_t result;
+ ssize_t size = readVInt32FromArray(&m_addr[m_position], &result);
+ if (size == -1) {
+ throw IOException("can't read vint32");
+ }
+ m_position += size;
+ return result;
}
- return i;
+ // We will probably run at the end of the stream, use the generic
+ // implementation and let readByte() handle that.
+ return InputStream::readVInt32();
}
View
73 src/util/vint.h
@@ -0,0 +1,73 @@
+// Copyright (C) 2011 Lukas Lalinsky
+// Distributed under the MIT license, see the LICENSE file for details.
+//
+// Some parts of this file are taken from the Protocol Buffers project:
+//
+// Copyright 2008 Google Inc. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef ACOUSTID_UTIL_VINT_H_
+#define ACOUSTID_UTIL_VINT_H_
+
+#include "common.h"
+
+namespace Acoustid {
+
+static const int kMaxVInt32Bytes = 5;
+static const int kMaxVIntBytes = kMaxVInt32Bytes; // only 32-bit vints
+
+inline const ssize_t readVInt32FromArray(const uint8_t *buffer, uint32_t *value)
+{
+ const uint8_t *ptr = buffer;
+ uint32_t b;
+ uint32_t result;
+
+ b = *(ptr++); result = (b & 0x7F) ; if (!(b & 0x80)) goto done;
+ b = *(ptr++); result |= (b & 0x7F) << 7; if (!(b & 0x80)) goto done;
+ b = *(ptr++); result |= (b & 0x7F) << 14; if (!(b & 0x80)) goto done;
+ b = *(ptr++); result |= (b & 0x7F) << 21; if (!(b & 0x80)) goto done;
+ b = *(ptr++); result |= b << 28; if (!(b & 0x80)) goto done;
+
+ // If the input is larger than 32 bits, we still need to read it all
+ // and discard the high-order bits.
+ for (int i = 0; i < kMaxVIntBytes - kMaxVInt32Bytes; i++) {
+ b = *(ptr++); if (!(b & 0x80)) goto done;
+ }
+
+ // We have overrun the maximum size of a varint (10 bytes). Assume
+ // the data is corrupt.
+ return -1;
+
+done:
+ *value = result;
+ return ptr - buffer;
+}
+
+}
+
+#endif
Please sign in to comment.
Something went wrong with that request. Please try again.