From 8a48ef58e2592336594bebb08a25ea763f83037c Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 18 Apr 2016 12:32:54 -0700 Subject: [PATCH 01/15] Duplicate IO C32 as explicitly unsigned CU32. Prepare for the addition of CI32 -- a signed compressed integer. --- core/Lucy/Store/InStream.c | 26 +++++++++++++++++++++ core/Lucy/Store/InStream.cfh | 17 +++++++++++--- core/Lucy/Store/OutStream.c | 30 ++++++++++++++++++++----- core/Lucy/Store/OutStream.cfh | 10 +++++++++ core/Lucy/Test/Store/TestIOPrimitives.c | 26 ++++++++++----------- 5 files changed, 87 insertions(+), 22 deletions(-) diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c index a6d61d88d..09da76fd3 100644 --- a/core/Lucy/Store/InStream.c +++ b/core/Lucy/Store/InStream.c @@ -49,6 +49,12 @@ S_fill(InStream *self, int64_t amount); static int64_t S_refill(InStream *self); +static CFISH_INLINE uint32_t +SI_read_cu32(InStream *self); + +static CFISH_INLINE uint64_t +SI_read_cu64(InStream *self); + InStream* InStream_open(Obj *file) { InStream *self = (InStream*)Class_Make_Obj(INSTREAM); @@ -468,6 +474,16 @@ InStream_Read_F64_IMP(InStream *self) { uint32_t InStream_Read_C32_IMP(InStream *self) { + return SI_read_cu32(self); +} + +uint32_t +InStream_Read_CU32_IMP(InStream *self) { + return SI_read_cu32(self); +} + +static CFISH_INLINE uint32_t +SI_read_cu32(InStream *self) { InStreamIVARS *const ivars = InStream_IVARS(self); uint32_t retval = 0; while (1) { @@ -482,6 +498,16 @@ InStream_Read_C32_IMP(InStream *self) { uint64_t InStream_Read_C64_IMP(InStream *self) { + return SI_read_cu64(self); +} + +uint64_t +InStream_Read_CU64_IMP(InStream *self) { + return SI_read_cu64(self); +} + +static CFISH_INLINE uint64_t +SI_read_cu64(InStream *self) { InStreamIVARS *const ivars = InStream_IVARS(self); uint64_t retval = 0; while (1) { diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh index efa7f6326..5fd5dabb3 100644 --- a/core/Lucy/Store/InStream.cfh +++ b/core/Lucy/Store/InStream.cfh @@ -176,15 +176,26 @@ class Lucy::Store::InStream inherits Clownfish::Obj { uint32_t Read_C32(InStream *self); + /** Read in a compressed 32-bit unsigned integer. + */ + uint32_t + Read_CU32(InStream *self); + /** Read a 64-bit integer, using the same encoding as a C32 but occupying * as many as 10 bytes. */ final uint64_t Read_C64(InStream *self); - /** Read the bytes for a C32/C64 into `buf`. Return the number - * of bytes read. The caller must ensure that sufficient space exists in - * `buf` (worst case is 10 bytes). + /** Read a 64-bit unsigned integer, using the same encoding as a CU32 but + * occupying as many as 10 bytes. + */ + final uint64_t + Read_CU64(InStream *self); + + /** Read the bytes for a compressed 64-bit integer into `buf`. Return the + * number of bytes read. The caller must ensure that sufficient space + * exists in `buf` (worst case is 10 bytes). */ final int Read_Raw_C64(InStream *self, char *buf); diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c index d0fadd4ae..8839791c8 100644 --- a/core/Lucy/Store/OutStream.c +++ b/core/Lucy/Store/OutStream.c @@ -33,9 +33,13 @@ static CFISH_INLINE void SI_write_bytes(OutStream *self, OutStreamIVARS *ivars, const void *bytes, size_t len); -// Inlined version of OutStream_Write_C32. +// Inlined version of OutStream_Write_CU32. static CFISH_INLINE void -SI_write_c32(OutStream *self, OutStreamIVARS *ivars, uint32_t value); +SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value); + +// Inlined version of OutStream_Write_CU64. +static CFISH_INLINE void +SI_write_cu64(OutStream *self, OutStreamIVARS *ivars, uint64_t value); // Flush content in the buffer to the FileHandle. static void @@ -287,11 +291,16 @@ OutStream_Write_F64_IMP(OutStream *self, double value) { void OutStream_Write_C32_IMP(OutStream *self, uint32_t value) { - SI_write_c32(self, OutStream_IVARS(self), value); + SI_write_cu32(self, OutStream_IVARS(self), value); +} + +void +OutStream_Write_CU32_IMP(OutStream *self, uint32_t value) { + SI_write_cu32(self, OutStream_IVARS(self), value); } static CFISH_INLINE void -SI_write_c32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) { +SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) { uint8_t buf[C32_MAX_BYTES]; uint8_t *ptr = buf + sizeof(buf) - 1; @@ -310,7 +319,16 @@ SI_write_c32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) { void OutStream_Write_C64_IMP(OutStream *self, uint64_t value) { - OutStreamIVARS *const ivars = OutStream_IVARS(self); + SI_write_cu64(self, OutStream_IVARS(self), value); +} + +void +OutStream_Write_CU64_IMP(OutStream *self, uint64_t value) { + SI_write_cu64(self, OutStream_IVARS(self), value); +} + +static CFISH_INLINE void +SI_write_cu64(OutStream *self, OutStreamIVARS *ivars, uint64_t value) { uint8_t buf[C64_MAX_BYTES]; uint8_t *ptr = buf + sizeof(buf) - 1; @@ -330,7 +348,7 @@ OutStream_Write_C64_IMP(OutStream *self, uint64_t value) { void OutStream_Write_String_IMP(OutStream *self, const char *string, size_t len) { OutStreamIVARS *const ivars = OutStream_IVARS(self); - SI_write_c32(self, ivars, (uint32_t)len); + SI_write_cu32(self, ivars, (uint32_t)len); SI_write_bytes(self, ivars, string, len); } diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh index 6bcacac81..372610155 100644 --- a/core/Lucy/Store/OutStream.cfh +++ b/core/Lucy/Store/OutStream.cfh @@ -118,11 +118,21 @@ class Lucy::Store::OutStream inherits Clownfish::Obj { final void Write_C32(OutStream *self, uint32_t value); + /** Write an unsigned 32-bit integer using a compressed format. + */ + final void + Write_CU32(OutStream *self, uint32_t value); + /** Write a 64-bit integer using a compressed format. */ final void Write_C64(OutStream *self, uint64_t value); + /** Write an unsigned 64-bit integer using a compressed format. + */ + final void + Write_CU64(OutStream *self, uint64_t value); + /** Write an IEEE 764 32-bit floating point number in big-endian byte * order. */ diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c index 551f1ac15..7409d7e42 100644 --- a/core/Lucy/Test/Store/TestIOPrimitives.c +++ b/core/Lucy/Test/Store/TestIOPrimitives.c @@ -242,7 +242,7 @@ test_u64(TestBatchRunner *runner) { } static void -test_c32(TestBatchRunner *runner) { +test_cu32(TestBatchRunner *runner) { uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT32_MAX); RAMFile *file = RAMFile_new(NULL, false); OutStream *outstream = OutStream_open((Obj*)file); @@ -256,21 +256,21 @@ test_c32(TestBatchRunner *runner) { ints[3] = UINT32_MAX - 1; for (i = 0; i < 1000; i++) { - OutStream_Write_C32(outstream, (uint32_t)ints[i]); + OutStream_Write_CU32(outstream, (uint32_t)ints[i]); } OutStream_Close(outstream); instream = InStream_open((Obj*)file); for (i = 0; i < 1000; i++) { - uint32_t got = InStream_Read_C32(instream); + uint32_t got = InStream_Read_CU32(instream); if (got != ints[i]) { - FAIL(runner, "c32 round trip failed: %lu, %lu", (unsigned long)got, + FAIL(runner, "cu32 round trip failed: %lu, %lu", (unsigned long)got, (unsigned long)ints[i]); break; } } if (i == 1000) { - PASS(runner, "c32 round trip"); + PASS(runner, "cu32 round trip"); } DECREF(instream); @@ -280,7 +280,7 @@ test_c32(TestBatchRunner *runner) { } static void -test_c64(TestBatchRunner *runner) { +test_cu64(TestBatchRunner *runner) { uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT64_MAX); RAMFile *file = RAMFile_new(NULL, false); RAMFile *raw_file = RAMFile_new(NULL, false); @@ -297,23 +297,23 @@ test_c64(TestBatchRunner *runner) { ints[3] = UINT64_MAX - 1; for (i = 0; i < 1000; i++) { - OutStream_Write_C64(outstream, ints[i]); - OutStream_Write_C64(raw_outstream, ints[i]); + OutStream_Write_CU64(outstream, ints[i]); + OutStream_Write_CU64(raw_outstream, ints[i]); } OutStream_Close(outstream); OutStream_Close(raw_outstream); instream = InStream_open((Obj*)file); for (i = 0; i < 1000; i++) { - uint64_t got = InStream_Read_C64(instream); + uint64_t got = InStream_Read_CU64(instream); if (got != ints[i]) { - FAIL(runner, "c64 round trip failed: %" PRIu64 ", %" PRIu64, + FAIL(runner, "cu64 round trip failed: %" PRIu64 ", %" PRIu64, got, ints[i]); break; } } if (i == 1000) { - PASS(runner, "c64 round trip"); + PASS(runner, "cu64 round trip"); } raw_instream = InStream_open((Obj*)raw_file); @@ -429,8 +429,8 @@ TestIOPrimitives_Run_IMP(TestIOPrimitives *self, TestBatchRunner *runner) { test_u32(runner); test_i64(runner); test_u64(runner); - test_c32(runner); - test_c64(runner); + test_cu32(runner); + test_cu64(runner); test_f32(runner); test_f64(runner); } From 3e3c64895bc92b19fadebe2a7b3188e8a218597d Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 18 Apr 2016 15:07:21 -0700 Subject: [PATCH 02/15] Add explicit cu32 and cu64 to NumUtil. Duplicate the functionality of xxxx_c64 in xxxx_cu64, in preparation for the addition of xxxx_ci64 (for signed integers) and the eventual removal of xxxx_64. --- core/Lucy/Test/Store/TestIOPrimitives.c | 2 +- core/Lucy/Test/Util/TestNumberUtils.c | 42 ++++++------- core/Lucy/Util/NumberUtils.cfh | 81 +++++++++++++++++++++++-- 3 files changed, 99 insertions(+), 26 deletions(-) diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c index 7409d7e42..3c09dde13 100644 --- a/core/Lucy/Test/Store/TestIOPrimitives.c +++ b/core/Lucy/Test/Store/TestIOPrimitives.c @@ -321,7 +321,7 @@ test_cu64(TestBatchRunner *runner) { char buffer[10]; const char *buf = buffer; size_t size = InStream_Read_Raw_C64(raw_instream, buffer); - uint64_t got = NumUtil_decode_c64(&buf); + uint64_t got = NumUtil_decode_cu64(&buf); UNUSED_VAR(size); if (got != ints[i]) { FAIL(runner, "Read_Raw_C64 failed: %" PRIu64 ", %" PRIu64, diff --git a/core/Lucy/Test/Util/TestNumberUtils.c b/core/Lucy/Test/Util/TestNumberUtils.c index 70a63a60c..ffd3079a9 100644 --- a/core/Lucy/Test/Util/TestNumberUtils.c +++ b/core/Lucy/Test/Util/TestNumberUtils.c @@ -94,14 +94,14 @@ test_u4(TestBatchRunner *runner) { } static void -test_c32(TestBatchRunner *runner) { +test_cu32(TestBatchRunner *runner) { uint64_t mins[] = { 0, 0x4000 - 100, (uint32_t)INT32_MAX - 100, UINT32_MAX - 10 }; uint64_t limits[] = { 500, 0x4000 + 100, (uint32_t)INT32_MAX + 100, UINT32_MAX }; uint32_t set_num; uint32_t num_sets = sizeof(mins) / sizeof(uint64_t); size_t count = 64; uint64_t *ints = NULL; - size_t amount = count * C32_MAX_BYTES; + size_t amount = count * CU32_MAX_BYTES; char *encoded = (char*)CALLOCATE(amount, sizeof(char)); char *target = encoded; char *limit = target + amount; @@ -114,13 +114,13 @@ test_c32(TestBatchRunner *runner) { target = encoded; for (size_t i = 0; i < count; i++) { ints[i] = (uint32_t)ints[i]; - NumUtil_encode_c32((uint32_t)ints[i], &target); + NumUtil_encode_cu32((uint32_t)ints[i], &target); } decode = encoded; skip = encoded; for (size_t i = 0; i < count; i++) { - TEST_INT_EQ(runner, NumUtil_decode_c32(&decode), ints[i], - "c32 %lu", (long)ints[i]); + TEST_INT_EQ(runner, NumUtil_decode_cu32(&decode), ints[i], + "cu32 %lu", (long)ints[i]); NumUtil_skip_cint(&skip); if (decode > limit) { THROW(ERR, "overrun"); } } @@ -129,16 +129,16 @@ test_c32(TestBatchRunner *runner) { target = encoded; for (size_t i = 0; i < count; i++) { - NumUtil_encode_padded_c32((uint32_t)ints[i], &target); + NumUtil_encode_padded_cu32((uint32_t)ints[i], &target); } TEST_TRUE(runner, target == limit, - "padded c32 uses 5 bytes (%lu == %lu)", (unsigned long)target, + "padded cu32 uses 5 bytes (%lu == %lu)", (unsigned long)target, (unsigned long)limit); decode = encoded; skip = encoded; for (size_t i = 0; i < count; i++) { - TEST_INT_EQ(runner, NumUtil_decode_c32(&decode), ints[i], - "padded c32 %lu", (long)ints[i]); + TEST_INT_EQ(runner, NumUtil_decode_cu32(&decode), ints[i], + "padded cu32 %lu", (long)ints[i]); NumUtil_skip_cint(&skip); if (decode > limit) { THROW(ERR, "overrun"); } } @@ -147,23 +147,23 @@ test_c32(TestBatchRunner *runner) { } target = encoded; - NumUtil_encode_c32(UINT32_MAX, &target); + NumUtil_encode_cu32(UINT32_MAX, &target); decode = encoded; - TEST_INT_EQ(runner, NumUtil_decode_c32(&decode), UINT32_MAX, "c32 UINT32_MAX"); + TEST_INT_EQ(runner, NumUtil_decode_cu32(&decode), UINT32_MAX, "cu32 UINT32_MAX"); FREEMEM(encoded); FREEMEM(ints); } static void -test_c64(TestBatchRunner *runner) { +test_cu64(TestBatchRunner *runner) { uint64_t mins[] = { 0, 0x4000 - 100, (uint64_t)UINT32_MAX - 100, UINT64_MAX - 10 }; uint64_t limits[] = { 500, 0x4000 + 100, (uint64_t)UINT32_MAX + 1000, UINT64_MAX }; uint32_t set_num; uint32_t num_sets = sizeof(mins) / sizeof(uint64_t); size_t count = 64; uint64_t *ints = NULL; - size_t amount = count * C64_MAX_BYTES; + size_t amount = count * CU64_MAX_BYTES; char *encoded = (char*)CALLOCATE(amount, sizeof(char)); char *target = encoded; char *limit = target + amount; @@ -175,14 +175,14 @@ test_c64(TestBatchRunner *runner) { mins[set_num], limits[set_num]); target = encoded; for (size_t i = 0; i < count; i++) { - NumUtil_encode_c64(ints[i], &target); + NumUtil_encode_cu64(ints[i], &target); } decode = encoded; skip = encoded; for (size_t i = 0; i < count; i++) { - uint64_t got = NumUtil_decode_c64(&decode); + uint64_t got = NumUtil_decode_cu64(&decode); TEST_TRUE(runner, got == ints[i], - "c64 %" PRIu64 " == %" PRIu64, got, ints[i]); + "cu64 %" PRIu64 " == %" PRIu64, got, ints[i]); if (decode > limit) { THROW(ERR, "overrun"); } NumUtil_skip_cint(&skip); } @@ -191,11 +191,11 @@ test_c64(TestBatchRunner *runner) { } target = encoded; - NumUtil_encode_c64(UINT64_MAX, &target); + NumUtil_encode_cu64(UINT64_MAX, &target); decode = encoded; - uint64_t got = NumUtil_decode_c64(&decode); - TEST_TRUE(runner, got == UINT64_MAX, "c64 UINT64_MAX"); + uint64_t got = NumUtil_decode_cu64(&decode); + TEST_TRUE(runner, got == UINT64_MAX, "cu64 UINT64_MAX"); FREEMEM(encoded); FREEMEM(ints); @@ -364,8 +364,8 @@ TestNumUtil_Run_IMP(TestNumberUtils *self, TestBatchRunner *runner) { test_u1(runner); test_u2(runner); test_u4(runner); - test_c32(runner); - test_c64(runner); + test_cu32(runner); + test_cu64(runner); test_bigend_u16(runner); test_bigend_u32(runner); test_bigend_u64(runner); diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh index b7c08b495..3d9c79d9a 100644 --- a/core/Lucy/Util/NumberUtils.cfh +++ b/core/Lucy/Util/NumberUtils.cfh @@ -87,6 +87,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_c32(uint32_t value, char **dest); + /** Encode a compressed 32-bit unsigned integer at the space pointed to by + * `dest`. As a side effect, `dest` will be advanced to immediately after + * the end of the compressed data. + */ + inert inline void + encode_cu32(uint32_t value, char **dest); + /** Encode a C32 at the space pointed to by `dest`, but add * "leading zeroes" so that the space consumed will always be 5 bytes. As * a side effect, `dest` will be advanced to immediately after @@ -95,6 +102,14 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_padded_c32(uint32_t value, char **dest); + /** Encode a compressed 32-bit unsigned integer at the space pointed to by + * `dest`, but add "leading zeroes" so that the space consumed will always + * be 5 bytes. As a side effect, `dest` will be advanced to immediately + * after the end of the compressed data. + */ + inert inline void + encode_padded_cu32(uint32_t value, char **dest); + /** Encode a C64 at the space pointed to by `dest`. As a side * effect, `dest` will be advanced to immediately after the end * of the C64. @@ -102,6 +117,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_c64(uint64_t value, char **dest); + /** Encode a compressed 64-bit unsigned integer at the space pointed to by + * `dest`. As a side effect, `dest` will be advanced to immediately after + * the end of the compressed data. + */ + inert inline void + encode_cu64(uint64_t value, char **dest); + /** Read a C32 from the buffer pointed to by `source`. As a * side effect, advance the pointer, consuming the bytes occupied by the * C32. @@ -109,6 +131,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline uint32_t decode_c32(const char **source); + /** Read a compressed 32-bit unsigned integer from the buffer pointed to + * by `source`. As a side effect, advance the pointer, consuming the + * bytes occupied by the compressed number. + */ + inert inline uint32_t + decode_cu32(const char **source); + /** Read a C64 from the buffer pointed to by `source`. As a * side effect, advance the pointer, consuming the bytes occupied by the * C64. @@ -116,6 +145,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline uint64_t decode_c64(const char **source); + /** Read a compressed 64-bit unsigned integer from the buffer pointed to + * by `source`. As a side effect, advance the pointer, consuming the + * bytes occupied by the compressed number. + */ + inert inline uint64_t + decode_cu64(const char **source); + /** Advance `source` past one encoded C32 or C64. */ inert inline void @@ -296,10 +332,17 @@ lucy_NumUtil_decode_bigend_f64(const void *source) { #define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */ #define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */ +#define LUCY_NUMUTIL_CU32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */ +#define LUCY_NUMUTIL_CU64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */ static CFISH_INLINE void lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) { - uint8_t buf[LUCY_NUMUTIL_C32_MAX_BYTES]; + lucy_NumUtil_encode_cu32(value, out_buf); +} + +static CFISH_INLINE void +lucy_NumUtil_encode_cu32(uint32_t value, char **out_buf) { + uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES]; uint8_t *const limit = buf + sizeof(buf); uint8_t *ptr = limit - 1; int num_bytes; @@ -318,7 +361,12 @@ lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) { static CFISH_INLINE void lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) { - uint8_t buf[LUCY_NUMUTIL_C64_MAX_BYTES]; + lucy_NumUtil_encode_cu64(value, out_buf); +} + +static CFISH_INLINE void +lucy_NumUtil_encode_cu64(uint64_t value, char **out_buf) { + uint8_t buf[LUCY_NUMUTIL_CU64_MAX_BYTES]; uint8_t *const limit = buf + sizeof(buf); uint8_t *ptr = limit - 1; int num_bytes; @@ -337,7 +385,12 @@ lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) { static CFISH_INLINE void lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) { - uint8_t buf[LUCY_NUMUTIL_C32_MAX_BYTES] + lucy_NumUtil_encode_padded_cu32(value, out_buf); +} + +static CFISH_INLINE void +lucy_NumUtil_encode_padded_cu32(uint32_t value, char **out_buf) { + uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES] = { 0x80, 0x80, 0x80, 0x80, 0x80 }; uint8_t *const limit = buf + sizeof(buf); uint8_t *ptr = limit - 1; @@ -349,7 +402,7 @@ lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) { *--ptr = ((value & 0x7f) | 0x80); value >>= 7; } - memcpy(*out_buf, buf, LUCY_NUMUTIL_C32_MAX_BYTES); + memcpy(*out_buf, buf, LUCY_NUMUTIL_CU32_MAX_BYTES); *out_buf += sizeof(buf); } @@ -371,6 +424,15 @@ lucy_NumUtil_decode_c32(const char **source_ptr) { return decoded; } +static CFISH_INLINE uint32_t +lucy_NumUtil_decode_cu32(const char **source_ptr) { + const char *source = *source_ptr; + uint32_t decoded; + LUCY_NUMUTIL_DECODE_CINT(decoded, source); + *source_ptr = source; + return decoded; +} + static CFISH_INLINE uint64_t lucy_NumUtil_decode_c64(const char **source_ptr) { const char *source = *source_ptr; @@ -380,6 +442,15 @@ lucy_NumUtil_decode_c64(const char **source_ptr) { return decoded; } +static CFISH_INLINE uint64_t +lucy_NumUtil_decode_cu64(const char **source_ptr) { + const char *source = *source_ptr; + uint64_t decoded; + LUCY_NUMUTIL_DECODE_CINT(decoded, source); + *source_ptr = source; + return decoded; +} + static CFISH_INLINE void lucy_NumUtil_skip_cint(const char **source_ptr) { const uint8_t *ptr = *(const uint8_t**)source_ptr; @@ -459,6 +530,8 @@ lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) { #ifdef LUCY_USE_SHORT_NAMES #define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES #define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES + #define CU32_MAX_BYTES LUCY_NUMUTIL_CU32_MAX_BYTES + #define CU64_MAX_BYTES LUCY_NUMUTIL_CU64_MAX_BYTES #endif __END_C__ From ce89b9c40ea86b45c181852a12edeaa7bea2d054 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 18 Apr 2016 15:43:38 -0700 Subject: [PATCH 03/15] Read/write compressed signed integers. --- core/Lucy/Store/InStream.c | 10 +++ core/Lucy/Store/InStream.cfh | 11 ++++ core/Lucy/Store/OutStream.c | 10 +++ core/Lucy/Store/OutStream.cfh | 10 +++ core/Lucy/Test/Store/TestIOPrimitives.c | 82 ++++++++++++++++++++++++- 5 files changed, 122 insertions(+), 1 deletion(-) diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c index 09da76fd3..d04652d94 100644 --- a/core/Lucy/Store/InStream.c +++ b/core/Lucy/Store/InStream.c @@ -477,6 +477,11 @@ InStream_Read_C32_IMP(InStream *self) { return SI_read_cu32(self); } +int32_t +InStream_Read_CI32_IMP(InStream *self) { + return (int32_t)SI_read_cu32(self); +} + uint32_t InStream_Read_CU32_IMP(InStream *self) { return SI_read_cu32(self); @@ -501,6 +506,11 @@ InStream_Read_C64_IMP(InStream *self) { return SI_read_cu64(self); } +int64_t +InStream_Read_CI64_IMP(InStream *self) { + return (int64_t)SI_read_cu64(self); +} + uint64_t InStream_Read_CU64_IMP(InStream *self) { return SI_read_cu64(self); diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh index 5fd5dabb3..921be7765 100644 --- a/core/Lucy/Store/InStream.cfh +++ b/core/Lucy/Store/InStream.cfh @@ -176,6 +176,11 @@ class Lucy::Store::InStream inherits Clownfish::Obj { uint32_t Read_C32(InStream *self); + /** Read in a compressed 32-bit signed integer. + */ + int32_t + Read_CI32(InStream *self); + /** Read in a compressed 32-bit unsigned integer. */ uint32_t @@ -187,6 +192,12 @@ class Lucy::Store::InStream inherits Clownfish::Obj { final uint64_t Read_C64(InStream *self); + /** Read a 64-bit signed integer, using the same encoding as a CI32 but + * occupying as many as 10 bytes. + */ + final int64_t + Read_CI64(InStream *self); + /** Read a 64-bit unsigned integer, using the same encoding as a CU32 but * occupying as many as 10 bytes. */ diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c index 8839791c8..3f1d94147 100644 --- a/core/Lucy/Store/OutStream.c +++ b/core/Lucy/Store/OutStream.c @@ -294,6 +294,11 @@ OutStream_Write_C32_IMP(OutStream *self, uint32_t value) { SI_write_cu32(self, OutStream_IVARS(self), value); } +void +OutStream_Write_CI32_IMP(OutStream *self, int32_t value) { + SI_write_cu32(self, OutStream_IVARS(self), (uint32_t)value); +} + void OutStream_Write_CU32_IMP(OutStream *self, uint32_t value) { SI_write_cu32(self, OutStream_IVARS(self), value); @@ -322,6 +327,11 @@ OutStream_Write_C64_IMP(OutStream *self, uint64_t value) { SI_write_cu64(self, OutStream_IVARS(self), value); } +void +OutStream_Write_CI64_IMP(OutStream *self, int64_t value) { + SI_write_cu64(self, OutStream_IVARS(self), (uint64_t)value); +} + void OutStream_Write_CU64_IMP(OutStream *self, uint64_t value) { SI_write_cu64(self, OutStream_IVARS(self), value); diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh index 372610155..cb6700369 100644 --- a/core/Lucy/Store/OutStream.cfh +++ b/core/Lucy/Store/OutStream.cfh @@ -118,6 +118,11 @@ class Lucy::Store::OutStream inherits Clownfish::Obj { final void Write_C32(OutStream *self, uint32_t value); + /** Write a signed 32-bit integer using a compressed format. + */ + final void + Write_CI32(OutStream *self, int32_t value); + /** Write an unsigned 32-bit integer using a compressed format. */ final void @@ -128,6 +133,11 @@ class Lucy::Store::OutStream inherits Clownfish::Obj { final void Write_C64(OutStream *self, uint64_t value); + /** Write a signed 64-bit integer using a compressed format. + */ + final void + Write_CI64(OutStream *self, int64_t value); + /** Write an unsigned 64-bit integer using a compressed format. */ final void diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c index 3c09dde13..8d0f76ff4 100644 --- a/core/Lucy/Test/Store/TestIOPrimitives.c +++ b/core/Lucy/Test/Store/TestIOPrimitives.c @@ -241,6 +241,45 @@ test_u64(TestBatchRunner *runner) { FREEMEM(ints); } +static void +test_ci32(TestBatchRunner *runner) { + int64_t *ints = TestUtils_random_i64s(NULL, 1000, INT32_MIN, INT32_MAX); + RAMFile *file = RAMFile_new(NULL, false); + OutStream *outstream = OutStream_open((Obj*)file); + InStream *instream; + size_t i; + + // Test boundaries. + ints[0] = 0; + ints[1] = 1; + ints[2] = -1; + ints[3] = INT32_MAX; + ints[4] = INT32_MIN; + + for (i = 0; i < 1000; i++) { + OutStream_Write_CI32(outstream, (int32_t)ints[i]); + } + OutStream_Close(outstream); + + instream = InStream_open((Obj*)file); + for (i = 0; i < 1000; i++) { + int32_t got = InStream_Read_CI32(instream); + if ((int64_t)got != ints[i]) { + FAIL(runner, "ci32 round trip failed: %" PRId32 ", %" PRId64, + got, ints[i]); + break; + } + } + if (i == 1000) { + PASS(runner, "ci32 round trip"); + } + + DECREF(instream); + DECREF(outstream); + DECREF(file); + FREEMEM(ints); +} + static void test_cu32(TestBatchRunner *runner) { uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT32_MAX); @@ -279,6 +318,45 @@ test_cu32(TestBatchRunner *runner) { FREEMEM(ints); } +static void +test_ci64(TestBatchRunner *runner) { + int64_t *ints = TestUtils_random_i64s(NULL, 1000, INT64_MIN, INT64_MAX); + RAMFile *file = RAMFile_new(NULL, false); + OutStream *outstream = OutStream_open((Obj*)file); + InStream *instream; + uint32_t i; + + // Test boundaries. + ints[0] = 0; + ints[1] = 1; + ints[2] = -1; + ints[3] = INT64_MAX; + ints[4] = INT64_MIN; + + for (i = 0; i < 1000; i++) { + OutStream_Write_CI64(outstream, ints[i]); + } + OutStream_Close(outstream); + + instream = InStream_open((Obj*)file); + for (i = 0; i < 1000; i++) { + int64_t got = InStream_Read_CI64(instream); + if (got != ints[i]) { + FAIL(runner, "ci64 round trip failed: %" PRId64 ", %" PRId64, + got, ints[i]); + break; + } + } + if (i == 1000) { + PASS(runner, "ci64 round trip"); + } + + DECREF(instream); + DECREF(outstream); + DECREF(file); + FREEMEM(ints); +} + static void test_cu64(TestBatchRunner *runner) { uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, UINT64_MAX); @@ -421,7 +499,7 @@ test_f64(TestBatchRunner *runner) { void TestIOPrimitives_Run_IMP(TestIOPrimitives *self, TestBatchRunner *runner) { - TestBatchRunner_Plan(runner, (TestBatch*)self, 11); + TestBatchRunner_Plan(runner, (TestBatch*)self, 13); srand((unsigned int)time((time_t*)NULL)); test_i8(runner); test_u8(runner); @@ -429,7 +507,9 @@ TestIOPrimitives_Run_IMP(TestIOPrimitives *self, TestBatchRunner *runner) { test_u32(runner); test_i64(runner); test_u64(runner); + test_ci32(runner); test_cu32(runner); + test_ci64(runner); test_cu64(runner); test_f32(runner); test_f64(runner); From cef8cbc1dc507196ac0881f72709031b2889abff Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 18 Apr 2016 17:12:14 -0700 Subject: [PATCH 04/15] Support compressed signed integers in NumberUtils. --- core/Lucy/Test/Util/TestNumberUtils.c | 104 +++++++++++++++++++++++++- core/Lucy/Util/NumberUtils.cfh | 52 +++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/core/Lucy/Test/Util/TestNumberUtils.c b/core/Lucy/Test/Util/TestNumberUtils.c index ffd3079a9..125f3255d 100644 --- a/core/Lucy/Test/Util/TestNumberUtils.c +++ b/core/Lucy/Test/Util/TestNumberUtils.c @@ -93,6 +93,55 @@ test_u4(TestBatchRunner *runner) { FREEMEM(ints); } +static void +test_ci32(TestBatchRunner *runner) { + int64_t mins[] = { -500, -0x4000 - 100, INT32_MIN }; + int64_t limits[] = { 500, -0x4000 + 100, INT32_MIN + 10 }; + int32_t set_num; + int32_t num_sets = sizeof(mins) / sizeof(int64_t); + size_t count = 64; + int64_t *ints = NULL; + size_t amount = count * CI32_MAX_BYTES; + char *encoded = (char*)CALLOCATE(amount, sizeof(char)); + char *target = encoded; + char *limit = target + amount; + const char *decode; + + for (set_num = 0; set_num < num_sets; set_num++) { + const char *skip; + ints = TestUtils_random_i64s(ints, count, + mins[set_num], limits[set_num]); + target = encoded; + for (size_t i = 0; i < count; i++) { + NumUtil_encode_ci32((int32_t)ints[i], &target); + } + decode = encoded; + skip = encoded; + for (size_t i = 0; i < count; i++) { + TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), ints[i], + "ci32 %" PRId64, ints[i]); + NumUtil_skip_cint(&skip); + if (decode > limit) { THROW(ERR, "overrun"); } + } + TEST_TRUE(runner, skip == decode, "skip %" PRIu64 " == %" PRIu64, + (uint64_t)skip, (uint64_t)decode); + } + + target = encoded; + NumUtil_encode_ci32(INT32_MAX, &target); + decode = encoded; + TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), INT32_MAX, + "ci32 INT32_MAX"); + target = encoded; + NumUtil_encode_ci32(INT32_MIN, &target); + decode = encoded; + TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), INT32_MIN, + "ci32 INT32_MIN"); + + FREEMEM(encoded); + FREEMEM(ints); +} + static void test_cu32(TestBatchRunner *runner) { uint64_t mins[] = { 0, 0x4000 - 100, (uint32_t)INT32_MAX - 100, UINT32_MAX - 10 }; @@ -155,6 +204,57 @@ test_cu32(TestBatchRunner *runner) { FREEMEM(ints); } +static void +test_ci64(TestBatchRunner *runner) { + int64_t mins[] = { -500, -0x4000 - 100, (int64_t)INT32_MIN - 100, INT64_MIN }; + int64_t limits[] = { 500, -0x4000 + 100, (int64_t)INT32_MIN + 1000, INT64_MIN + 10 }; + int32_t set_num; + int32_t num_sets = sizeof(mins) / sizeof(int64_t); + size_t count = 64; + int64_t *ints = NULL; + size_t amount = count * CI64_MAX_BYTES; + char *encoded = (char*)CALLOCATE(amount, sizeof(char)); + char *target = encoded; + char *limit = target + amount; + const char *decode; + + for (set_num = 0; set_num < num_sets; set_num++) { + const char *skip; + ints = TestUtils_random_i64s(ints, count, + mins[set_num], limits[set_num]); + target = encoded; + for (size_t i = 0; i < count; i++) { + NumUtil_encode_ci64(ints[i], &target); + } + decode = encoded; + skip = encoded; + for (size_t i = 0; i < count; i++) { + int64_t got = NumUtil_decode_ci64(&decode); + TEST_INT_EQ(runner, got, ints[i], + "ci64 %" PRId64 " == %" PRId64, got, ints[i]); + if (decode > limit) { THROW(ERR, "overrun"); } + NumUtil_skip_cint(&skip); + } + TEST_TRUE(runner, skip == decode, "skip %lu == %lu", + (unsigned long)skip, (unsigned long)decode); + } + + target = encoded; + NumUtil_encode_ci64(INT64_MAX, &target); + decode = encoded; + int64_t got = NumUtil_decode_ci64(&decode); + TEST_INT_EQ(runner, got, INT64_MAX, "ci64 INT64_MAX"); + + target = encoded; + NumUtil_encode_ci64(INT64_MIN, &target); + decode = encoded; + got = NumUtil_decode_ci64(&decode); + TEST_INT_EQ(runner, got, INT64_MIN, "ci64 INT64_MIN"); + + FREEMEM(encoded); + FREEMEM(ints); +} + static void test_cu64(TestBatchRunner *runner) { uint64_t mins[] = { 0, 0x4000 - 100, (uint64_t)UINT32_MAX - 100, UINT64_MAX - 10 }; @@ -359,12 +459,14 @@ test_bigend_f64(TestBatchRunner *runner) { void TestNumUtil_Run_IMP(TestNumberUtils *self, TestBatchRunner *runner) { - TestBatchRunner_Plan(runner, (TestBatch*)self, 1196); + TestBatchRunner_Plan(runner, (TestBatch*)self, 1655); srand((unsigned int)time((time_t*)NULL)); test_u1(runner); test_u2(runner); test_u4(runner); + test_ci32(runner); test_cu32(runner); + test_ci64(runner); test_cu64(runner); test_bigend_u16(runner); test_bigend_u32(runner); diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh index 3d9c79d9a..8d1b0b1de 100644 --- a/core/Lucy/Util/NumberUtils.cfh +++ b/core/Lucy/Util/NumberUtils.cfh @@ -87,6 +87,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_c32(uint32_t value, char **dest); + /** Encode a compressed 32-bit signed integer at the space pointed to by + * `dest`. As a side effect, `dest` will be advanced to immediately after + * the end of the compressed data. + */ + inert inline void + encode_ci32(int32_t value, char **dest); + /** Encode a compressed 32-bit unsigned integer at the space pointed to by * `dest`. As a side effect, `dest` will be advanced to immediately after * the end of the compressed data. @@ -117,6 +124,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_c64(uint64_t value, char **dest); + /** Encode a compressed 64-bit signed integer at the space pointed to by + * `dest`. As a side effect, `dest` will be advanced to immediately after + * the end of the compressed data. + */ + inert inline void + encode_ci64(int64_t value, char **dest); + /** Encode a compressed 64-bit unsigned integer at the space pointed to by * `dest`. As a side effect, `dest` will be advanced to immediately after * the end of the compressed data. @@ -131,6 +145,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline uint32_t decode_c32(const char **source); + /** Read a compressed 32-bit signed integer from the buffer pointed to + * by `source`. As a side effect, advance the pointer, consuming the + * bytes occupied by the compressed number. + */ + inert inline int32_t + decode_ci32(const char **source); + /** Read a compressed 32-bit unsigned integer from the buffer pointed to * by `source`. As a side effect, advance the pointer, consuming the * bytes occupied by the compressed number. @@ -145,6 +166,13 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline uint64_t decode_c64(const char **source); + /** Read a compressed 64-bit signed integer from the buffer pointed to + * by `source`. As a side effect, advance the pointer, consuming the + * bytes occupied by the compressed number. + */ + inert inline int64_t + decode_ci64(const char **source); + /** Read a compressed 64-bit unsigned integer from the buffer pointed to * by `source`. As a side effect, advance the pointer, consuming the * bytes occupied by the compressed number. @@ -332,7 +360,9 @@ lucy_NumUtil_decode_bigend_f64(const void *source) { #define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */ #define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */ +#define LUCY_NUMUTIL_CI32_MAX_BYTES ((sizeof(int32_t) * 8 / 7) + 1) /* 5 */ #define LUCY_NUMUTIL_CU32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */ +#define LUCY_NUMUTIL_CI64_MAX_BYTES ((sizeof(int64_t) * 8 / 7) + 1) /* 10 */ #define LUCY_NUMUTIL_CU64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */ static CFISH_INLINE void @@ -340,6 +370,11 @@ lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) { lucy_NumUtil_encode_cu32(value, out_buf); } +static CFISH_INLINE void +lucy_NumUtil_encode_ci32(int32_t value, char **out_buf) { + lucy_NumUtil_encode_cu32((uint32_t)value, out_buf); +} + static CFISH_INLINE void lucy_NumUtil_encode_cu32(uint32_t value, char **out_buf) { uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES]; @@ -364,6 +399,11 @@ lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) { lucy_NumUtil_encode_cu64(value, out_buf); } +static CFISH_INLINE void +lucy_NumUtil_encode_ci64(int64_t value, char **out_buf) { + lucy_NumUtil_encode_cu64((uint64_t)value, out_buf); +} + static CFISH_INLINE void lucy_NumUtil_encode_cu64(uint64_t value, char **out_buf) { uint8_t buf[LUCY_NUMUTIL_CU64_MAX_BYTES]; @@ -424,6 +464,11 @@ lucy_NumUtil_decode_c32(const char **source_ptr) { return decoded; } +static CFISH_INLINE int32_t +lucy_NumUtil_decode_ci32(const char **source_ptr) { + return (int32_t)lucy_NumUtil_decode_cu32(source_ptr); +} + static CFISH_INLINE uint32_t lucy_NumUtil_decode_cu32(const char **source_ptr) { const char *source = *source_ptr; @@ -442,6 +487,11 @@ lucy_NumUtil_decode_c64(const char **source_ptr) { return decoded; } +static CFISH_INLINE int64_t +lucy_NumUtil_decode_ci64(const char **source_ptr) { + return (int64_t)lucy_NumUtil_decode_cu64(source_ptr); +} + static CFISH_INLINE uint64_t lucy_NumUtil_decode_cu64(const char **source_ptr) { const char *source = *source_ptr; @@ -530,6 +580,8 @@ lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) { #ifdef LUCY_USE_SHORT_NAMES #define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES #define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES + #define CI32_MAX_BYTES LUCY_NUMUTIL_CI32_MAX_BYTES + #define CI64_MAX_BYTES LUCY_NUMUTIL_CI64_MAX_BYTES #define CU32_MAX_BYTES LUCY_NUMUTIL_CU32_MAX_BYTES #define CU64_MAX_BYTES LUCY_NUMUTIL_CU64_MAX_BYTES #endif From 67656d94224b9350deb224699eda5956a0b8a989 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 18 Apr 2016 17:52:42 -0700 Subject: [PATCH 05/15] Switch NumUtils client code to cu32/ci32. Change all invocations from NumUtil_xxxx_c32 to use the NumUtil_xxxx_cu32 and NumUtil_xxx_ci32 variants which are explicitly signed or unsigned. --- core/Lucy/Index/DocVector.c | 18 +++++++++--------- core/Lucy/Index/HighlightWriter.c | 26 +++++++++++++------------- core/Lucy/Index/Posting/RichPosting.c | 8 ++++---- core/Lucy/Index/Posting/ScorePosting.c | 18 +++++++++--------- core/Lucy/Store/OutStream.c | 4 ++-- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/core/Lucy/Index/DocVector.c b/core/Lucy/Index/DocVector.c index 54dfedd40..2bc819232 100644 --- a/core/Lucy/Index/DocVector.c +++ b/core/Lucy/Index/DocVector.c @@ -122,13 +122,13 @@ static Hash* S_extract_tv_cache(Blob *field_buf) { Hash *tv_cache = Hash_new(0); const char *tv_string = Blob_Get_Buf(field_buf); - int32_t num_terms = NumUtil_decode_c32(&tv_string); + int32_t num_terms = NumUtil_decode_ci32(&tv_string); ByteBuf *text_buf = BB_new(0); // Read the number of highlightable terms in the field. for (int32_t i = 0; i < num_terms; i++) { - size_t overlap = NumUtil_decode_c32(&tv_string); - size_t len = NumUtil_decode_c32(&tv_string); + size_t overlap = NumUtil_decode_cu32(&tv_string); + size_t len = NumUtil_decode_cu32(&tv_string); // Decompress the term text. BB_Set_Size(text_buf, overlap); @@ -137,7 +137,7 @@ S_extract_tv_cache(Blob *field_buf) { // Get positions & offsets string. const char *bookmark_ptr = tv_string; - int32_t num_positions = NumUtil_decode_c32(&tv_string); + int32_t num_positions = NumUtil_decode_ci32(&tv_string); while (num_positions--) { // Leave nums compressed to save a little mem. NumUtil_skip_cint(&tv_string); @@ -167,17 +167,17 @@ S_extract_tv_from_tv_buf(String *field, String *term_text, Blob *tv_buf) { uint32_t num_pos = 0; if (posdata != posdata_end) { - num_pos = NumUtil_decode_c32(&posdata); + num_pos = NumUtil_decode_cu32(&posdata); positions = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); } - // Expand C32s. + // Expand CI32s. for (uint32_t i = 0; i < num_pos; i++) { - positions[i] = NumUtil_decode_c32(&posdata); - starts[i] = NumUtil_decode_c32(&posdata); - ends[i] = NumUtil_decode_c32(&posdata); + positions[i] = NumUtil_decode_ci32(&posdata); + starts[i] = NumUtil_decode_ci32(&posdata); + ends[i] = NumUtil_decode_ci32(&posdata); } if (posdata != posdata_end) { diff --git a/core/Lucy/Index/HighlightWriter.c b/core/Lucy/Index/HighlightWriter.c index 08ad85a88..d81b623db 100644 --- a/core/Lucy/Index/HighlightWriter.c +++ b/core/Lucy/Index/HighlightWriter.c @@ -149,8 +149,8 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) { uint32_t freq; UNUSED_VAR(self); - // Leave space for a c32 indicating the number of postings. - BB_Set_Size(tv_buf, C32_MAX_BYTES); + // Leave space for a cu32 indicating the number of postings. + BB_Set_Size(tv_buf, CU32_MAX_BYTES); Inversion_Reset(inversion); while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) { @@ -164,11 +164,11 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) { char *orig; size_t old_size = BB_Get_Size(tv_buf); size_t new_size = old_size - + C32_MAX_BYTES // overlap - + C32_MAX_BYTES // length of string diff + + CI32_MAX_BYTES // overlap + + CI32_MAX_BYTES // length of string diff + (token_len - overlap) // diff char data - + C32_MAX_BYTES // num prox - + (C32_MAX_BYTES * freq * 3); // pos data + + CU32_MAX_BYTES // num prox + + (CU32_MAX_BYTES * freq * 3); // pos data // Allocate for worst-case scenario. ptr = BB_Grow(tv_buf, new_size); @@ -179,8 +179,8 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) { num_postings += 1; // Append the string diff to the tv_buf. - NumUtil_encode_c32(overlap, &ptr); - NumUtil_encode_c32((token_len - overlap), &ptr); + NumUtil_encode_ci32(overlap, &ptr); + NumUtil_encode_ci32((token_len - overlap), &ptr); memcpy(ptr, (token_text + overlap), (token_len - overlap)); ptr += token_len - overlap; @@ -189,13 +189,13 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) { last_len = token_len; // Append the number of positions for this term. - NumUtil_encode_c32(freq, &ptr); + NumUtil_encode_cu32(freq, &ptr); do { // Add position, start_offset, and end_offset to tv_buf. - NumUtil_encode_c32(Token_Get_Pos(token), &ptr); - NumUtil_encode_c32(Token_Get_Start_Offset(token), &ptr); - NumUtil_encode_c32(Token_Get_End_Offset(token), &ptr); + NumUtil_encode_ci32(Token_Get_Pos(token), &ptr); + NumUtil_encode_cu32(Token_Get_Start_Offset(token), &ptr); + NumUtil_encode_cu32(Token_Get_End_Offset(token), &ptr); } while (--freq && (token = *++tokens)); // Set new byte length. @@ -204,7 +204,7 @@ HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) { // Go back and start the term vector string with the posting count. char *dest = BB_Get_Buf(tv_buf); - NumUtil_encode_padded_c32(num_postings, &dest); + NumUtil_encode_padded_cu32(num_postings, &dest); Blob *blob = BB_Yield_Blob(tv_buf); DECREF(tv_buf); diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c index 17e3e95ce..b3ff7e42d 100644 --- a/core/Lucy/Index/Posting/RichPosting.c +++ b/core/Lucy/Index/Posting/RichPosting.c @@ -33,12 +33,12 @@ #include "Lucy/Util/MemoryPool.h" #include "Lucy/Util/NumberUtils.h" -#define FREQ_MAX_LEN C32_MAX_BYTES +#define FREQ_MAX_LEN CU32_MAX_BYTES #define MAX_RAW_POSTING_LEN(_raw_posting_size, _text_len, _freq) \ ( _raw_posting_size \ + _text_len /* term text content */ \ - + FREQ_MAX_LEN /* freq c32 */ \ - + (C32_MAX_BYTES * _freq) /* positions deltas */ \ + + FREQ_MAX_LEN /* freq cu32 */ \ + + (CU32_MAX_BYTES * _freq) /* positions deltas */ \ + _freq /* per-pos boost byte */ \ ) @@ -137,7 +137,7 @@ RichPost_Add_Inversion_To_Pool_IMP(RichPosting *self, PostingPool *post_pool, const uint32_t prox_delta = t_ivars->pos - last_prox; const float boost = field_boost * t_ivars->boost; - NumUtil_encode_c32(prox_delta, &dest); + NumUtil_encode_cu32(prox_delta, &dest); last_prox = t_ivars->pos; *((uint8_t*)dest) = Sim_Encode_Norm(sim, boost); diff --git a/core/Lucy/Index/Posting/ScorePosting.c b/core/Lucy/Index/Posting/ScorePosting.c index 3e995a8b0..bc84bf88a 100644 --- a/core/Lucy/Index/Posting/ScorePosting.c +++ b/core/Lucy/Index/Posting/ScorePosting.c @@ -35,13 +35,13 @@ #include "Lucy/Util/NumberUtils.h" #define FIELD_BOOST_LEN 1 -#define FREQ_MAX_LEN C32_MAX_BYTES +#define FREQ_MAX_LEN CU32_MAX_BYTES #define MAX_RAW_POSTING_LEN(_raw_post_size, _text_len, _freq) \ ( _raw_post_size \ + _text_len /* term text content */ \ + FIELD_BOOST_LEN /* field boost byte */ \ - + FREQ_MAX_LEN /* freq c32 */ \ - + (C32_MAX_BYTES * _freq) /* positions deltas */ \ + + FREQ_MAX_LEN /* freq cu32 */ \ + + (CU32_MAX_BYTES * _freq) /* positions deltas */ \ ) ScorePosting* @@ -110,7 +110,7 @@ ScorePost_Add_Inversion_To_Pool_IMP(ScorePosting *self, for (uint32_t i = 0; i < freq; i++) { TokenIVARS *const t_ivars = Token_IVARS(tokens[i]); const uint32_t prox_delta = t_ivars->pos - last_prox; - NumUtil_encode_c32(prox_delta, &dest); + NumUtil_encode_cu32(prox_delta, &dest); last_prox = t_ivars->pos; } @@ -134,9 +134,9 @@ void ScorePost_Read_Record_IMP(ScorePosting *self, InStream *instream) { ScorePostingIVARS *const ivars = ScorePost_IVARS(self); uint32_t position = 0; - const size_t max_start_bytes = (C32_MAX_BYTES * 2) + 1; + const size_t max_start_bytes = (CU32_MAX_BYTES * 2) + 1; const char *buf = InStream_Buf(instream, max_start_bytes); - const uint32_t doc_code = NumUtil_decode_c32(&buf); + const uint32_t doc_code = NumUtil_decode_cu32(&buf); const uint32_t doc_delta = doc_code >> 1; // Apply delta doc and retrieve freq. @@ -145,7 +145,7 @@ ScorePost_Read_Record_IMP(ScorePosting *self, InStream *instream) { ivars->freq = 1; } else { - ivars->freq = NumUtil_decode_c32(&buf); + ivars->freq = NumUtil_decode_cu32(&buf); } // Decode boost/norm byte. @@ -162,9 +162,9 @@ ScorePost_Read_Record_IMP(ScorePosting *self, InStream *instream) { uint32_t *positions = ivars->prox; InStream_Advance_Buf(instream, buf); - buf = InStream_Buf(instream, num_prox * C32_MAX_BYTES); + buf = InStream_Buf(instream, num_prox * CU32_MAX_BYTES); while (num_prox--) { - position += NumUtil_decode_c32(&buf); + position += NumUtil_decode_cu32(&buf); *positions++ = position; } diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c index 3f1d94147..8ae07d42f 100644 --- a/core/Lucy/Store/OutStream.c +++ b/core/Lucy/Store/OutStream.c @@ -306,7 +306,7 @@ OutStream_Write_CU32_IMP(OutStream *self, uint32_t value) { static CFISH_INLINE void SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) { - uint8_t buf[C32_MAX_BYTES]; + uint8_t buf[CU32_MAX_BYTES]; uint8_t *ptr = buf + sizeof(buf) - 1; // Write last byte first, which has no continue bit. @@ -339,7 +339,7 @@ OutStream_Write_CU64_IMP(OutStream *self, uint64_t value) { static CFISH_INLINE void SI_write_cu64(OutStream *self, OutStreamIVARS *ivars, uint64_t value) { - uint8_t buf[C64_MAX_BYTES]; + uint8_t buf[CU64_MAX_BYTES]; uint8_t *ptr = buf + sizeof(buf) - 1; // Write last byte first, which has no continue bit. From c1af09331739ee47e4a9dc138b7dd0265ba74497 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 18 Apr 2016 18:45:19 -0700 Subject: [PATCH 06/15] Remove obsolete NumUtil C32/C64 code. --- core/Lucy/Util/NumberUtils.cfh | 73 ---------------------------------- 1 file changed, 73 deletions(-) diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh index 8d1b0b1de..792890622 100644 --- a/core/Lucy/Util/NumberUtils.cfh +++ b/core/Lucy/Util/NumberUtils.cfh @@ -80,13 +80,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline double decode_bigend_f64(const void *source); - /** Encode a C32 at the space pointed to by `dest`. As a side - * effect, `dest` will be advanced to immediately after the end - * of the C32. - */ - inert inline void - encode_c32(uint32_t value, char **dest); - /** Encode a compressed 32-bit signed integer at the space pointed to by * `dest`. As a side effect, `dest` will be advanced to immediately after * the end of the compressed data. @@ -101,14 +94,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_cu32(uint32_t value, char **dest); - /** Encode a C32 at the space pointed to by `dest`, but add - * "leading zeroes" so that the space consumed will always be 5 bytes. As - * a side effect, `dest` will be advanced to immediately after - * the end of the C32. - */ - inert inline void - encode_padded_c32(uint32_t value, char **dest); - /** Encode a compressed 32-bit unsigned integer at the space pointed to by * `dest`, but add "leading zeroes" so that the space consumed will always * be 5 bytes. As a side effect, `dest` will be advanced to immediately @@ -117,13 +102,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_padded_cu32(uint32_t value, char **dest); - /** Encode a C64 at the space pointed to by `dest`. As a side - * effect, `dest` will be advanced to immediately after the end - * of the C64. - */ - inert inline void - encode_c64(uint64_t value, char **dest); - /** Encode a compressed 64-bit signed integer at the space pointed to by * `dest`. As a side effect, `dest` will be advanced to immediately after * the end of the compressed data. @@ -138,13 +116,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline void encode_cu64(uint64_t value, char **dest); - /** Read a C32 from the buffer pointed to by `source`. As a - * side effect, advance the pointer, consuming the bytes occupied by the - * C32. - */ - inert inline uint32_t - decode_c32(const char **source); - /** Read a compressed 32-bit signed integer from the buffer pointed to * by `source`. As a side effect, advance the pointer, consuming the * bytes occupied by the compressed number. @@ -159,13 +130,6 @@ inert class Lucy::Util::NumberUtils nickname NumUtil { inert inline uint32_t decode_cu32(const char **source); - /** Read a C64 from the buffer pointed to by `source`. As a - * side effect, advance the pointer, consuming the bytes occupied by the - * C64. - */ - inert inline uint64_t - decode_c64(const char **source); - /** Read a compressed 64-bit signed integer from the buffer pointed to * by `source`. As a side effect, advance the pointer, consuming the * bytes occupied by the compressed number. @@ -358,18 +322,11 @@ lucy_NumUtil_decode_bigend_f64(const void *source) { return duo.d; } -#define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */ -#define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */ #define LUCY_NUMUTIL_CI32_MAX_BYTES ((sizeof(int32_t) * 8 / 7) + 1) /* 5 */ #define LUCY_NUMUTIL_CU32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) /* 5 */ #define LUCY_NUMUTIL_CI64_MAX_BYTES ((sizeof(int64_t) * 8 / 7) + 1) /* 10 */ #define LUCY_NUMUTIL_CU64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) /* 10 */ -static CFISH_INLINE void -lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) { - lucy_NumUtil_encode_cu32(value, out_buf); -} - static CFISH_INLINE void lucy_NumUtil_encode_ci32(int32_t value, char **out_buf) { lucy_NumUtil_encode_cu32((uint32_t)value, out_buf); @@ -394,11 +351,6 @@ lucy_NumUtil_encode_cu32(uint32_t value, char **out_buf) { *out_buf += num_bytes; } -static CFISH_INLINE void -lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) { - lucy_NumUtil_encode_cu64(value, out_buf); -} - static CFISH_INLINE void lucy_NumUtil_encode_ci64(int64_t value, char **out_buf) { lucy_NumUtil_encode_cu64((uint64_t)value, out_buf); @@ -423,11 +375,6 @@ lucy_NumUtil_encode_cu64(uint64_t value, char **out_buf) { *out_buf += num_bytes; } -static CFISH_INLINE void -lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) { - lucy_NumUtil_encode_padded_cu32(value, out_buf); -} - static CFISH_INLINE void lucy_NumUtil_encode_padded_cu32(uint32_t value, char **out_buf) { uint8_t buf[LUCY_NUMUTIL_CU32_MAX_BYTES] @@ -455,15 +402,6 @@ lucy_NumUtil_encode_padded_cu32(uint32_t value, char **out_buf) { } \ } while (0) -static CFISH_INLINE uint32_t -lucy_NumUtil_decode_c32(const char **source_ptr) { - const char *source = *source_ptr; - uint32_t decoded; - LUCY_NUMUTIL_DECODE_CINT(decoded, source); - *source_ptr = source; - return decoded; -} - static CFISH_INLINE int32_t lucy_NumUtil_decode_ci32(const char **source_ptr) { return (int32_t)lucy_NumUtil_decode_cu32(source_ptr); @@ -478,15 +416,6 @@ lucy_NumUtil_decode_cu32(const char **source_ptr) { return decoded; } -static CFISH_INLINE uint64_t -lucy_NumUtil_decode_c64(const char **source_ptr) { - const char *source = *source_ptr; - uint64_t decoded; - LUCY_NUMUTIL_DECODE_CINT(decoded, source); - *source_ptr = source; - return decoded; -} - static CFISH_INLINE int64_t lucy_NumUtil_decode_ci64(const char **source_ptr) { return (int64_t)lucy_NumUtil_decode_cu64(source_ptr); @@ -578,8 +507,6 @@ lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) { } #ifdef LUCY_USE_SHORT_NAMES - #define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES - #define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES #define CI32_MAX_BYTES LUCY_NUMUTIL_CI32_MAX_BYTES #define CI64_MAX_BYTES LUCY_NUMUTIL_CI64_MAX_BYTES #define CU32_MAX_BYTES LUCY_NUMUTIL_CU32_MAX_BYTES From cbe964dfa85720357b8e6f06347028671c641e06 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Tue, 19 Apr 2016 18:59:15 -0700 Subject: [PATCH 07/15] Switch Perl bindings from C32/C64 to CI32/etc. Change IO methods invoked to use explicitly signed/unsigned versions rather than casting. --- perl/buildlib/Lucy/Build/Binding/Store.pm | 7 +++++-- perl/lib/LucyX/Search/Filter.pm | 4 ++-- perl/t/023-stepper.t | 4 ++-- perl/t/102-strings_io.t | 2 +- perl/t/binding/101-simple_io.t | 20 ++++++++++---------- perl/xs/Lucy/Document/Doc.c | 8 ++++---- perl/xs/Lucy/Index/DocReader.c | 14 +++++++------- 7 files changed, 31 insertions(+), 28 deletions(-) diff --git a/perl/buildlib/Lucy/Build/Binding/Store.pm b/perl/buildlib/Lucy/Build/Binding/Store.pm index a266aced2..f5196c85d 100644 --- a/perl/buildlib/Lucy/Build/Binding/Store.pm +++ b/perl/buildlib/Lucy/Build/Binding/Store.pm @@ -175,7 +175,7 @@ read_string(self) CODE: { char *ptr; - size_t len = LUCY_InStream_Read_C32(self); + size_t len = LUCY_InStream_Read_CU32(self); RETVAL = newSV(len + 1); SvCUR_set(RETVAL, len); SvPOK_on(RETVAL); @@ -364,7 +364,10 @@ PPCODE: { STRLEN len = 0; char *ptr = SvPVutf8(aSV, len); - LUCY_OutStream_Write_C32(self, len); + if (len > INT32_MAX) { + CFISH_THROW(CFISH_ERR, "String too long: %u64", (uint64_t)len); + } + LUCY_OutStream_Write_CU32(self, len); LUCY_OutStream_Write_Bytes(self, ptr, len); } END_XS_CODE diff --git a/perl/lib/LucyX/Search/Filter.pm b/perl/lib/LucyX/Search/Filter.pm index d168e9e8f..d6db91fb8 100644 --- a/perl/lib/LucyX/Search/Filter.pm +++ b/perl/lib/LucyX/Search/Filter.pm @@ -63,14 +63,14 @@ sub serialize { my ( $self, $outstream ) = @_; $self->SUPER::serialize($outstream); my $frozen = nfreeze( $query{$$self} ); - $outstream->write_c32( bytes::length($frozen) ); + $outstream->write_cu32( bytes::length($frozen) ); $outstream->print($frozen); } sub deserialize { my ( $self, $instream ) = @_; $self->SUPER::deserialize($instream); - my $len = $instream->read_c32; + my $len = $instream->read_cu32; my $frozen; $instream->read( $frozen, $len ); $query{$$self} = thaw($frozen); diff --git a/perl/t/023-stepper.t b/perl/t/023-stepper.t index 7ea2fa83a..2786ad70d 100644 --- a/perl/t/023-stepper.t +++ b/perl/t/023-stepper.t @@ -38,7 +38,7 @@ sub get_number { $number{ ${ +shift } } } sub read_record { my ( $self, $instream ) = @_; - $number{$$self} += $instream->read_c32; + $number{$$self} += $instream->read_ci32; } package main; @@ -47,7 +47,7 @@ use Lucy::Test; my $folder = Lucy::Store::RAMFolder->new; my $outstream = $folder->open_out("foo") or die Clownfish->error; -$outstream->write_c32(10) for 1 .. 5; +$outstream->write_ci32(10) for 1 .. 5; $outstream->close; my $instream = $folder->open_in("foo") or die Clownfish->error; my $stepper = MyStepper->new; diff --git a/perl/t/102-strings_io.t b/perl/t/102-strings_io.t index 657bf46c6..02d7ca839 100644 --- a/perl/t/102-strings_io.t +++ b/perl/t/102-strings_io.t @@ -24,7 +24,7 @@ my ( @items, $packed, $template, $buf, $file, $out, $in, $correct ); $file = Lucy::Store::RAMFile->new; $out = Lucy::Store::OutStream->open( file => $file ) or die Clownfish->error; -$out->write_c64(10000); +$out->write_cu64(10000); $out->close; $in = Lucy::Store::InStream->open( file => $file ) or die Clownfish->error; diff --git a/perl/t/binding/101-simple_io.t b/perl/t/binding/101-simple_io.t index fd5a9bed9..85db57a57 100644 --- a/perl/t/binding/101-simple_io.t +++ b/perl/t/binding/101-simple_io.t @@ -47,7 +47,7 @@ sub check_round_trip_bytes { my $file = Lucy::Store::RAMFile->new; my $outstream = Lucy::Store::OutStream->open( file => $file ); for (@$expected) { - $outstream->write_c32( bytes::length($_) ); + $outstream->write_ci32( bytes::length($_) ); $outstream->print($_); } $outstream->close; @@ -56,7 +56,7 @@ sub check_round_trip_bytes { my @got; for (@$expected) { my $buf; - my $len = $instream->read_c32; + my $len = $instream->read_ci32; $instream->read( $buf, $len ); push @got, $buf; } @@ -95,24 +95,24 @@ $_ += int( rand( 2**16 ) ) for @nums; check_round_trip( 'u64', \@nums ); @nums = ( 0 .. 127 ); -check_round_trip( 'c32', \@nums ); +check_round_trip( 'ci32', \@nums ); @nums = ( 128 .. 500 ); $packed = pack( 'w*', @nums ); -$ram_file = check_round_trip( 'c32', \@nums ); -is( $ram_file->get_contents, $packed, "C32 is equivalent to Perl's pack w" ); +$ram_file = check_round_trip( 'cu32', \@nums ); +is( $ram_file->get_contents, $packed, "CU32 is equivalent to Perl's pack w" ); @nums = ( 0 .. 127 ); -check_round_trip( 'c64', \@nums ); +check_round_trip( 'ci64', \@nums ); @nums = ( 128 .. 500 ); $packed = pack( 'w*', @nums ); -$ram_file = check_round_trip( 'c64', \@nums ); -is( $ram_file->get_contents, $packed, "C64 is equivalent to Perl's pack w" ); +$ram_file = check_round_trip( 'cu64', \@nums ); +is( $ram_file->get_contents, $packed, "CU64 is equivalent to Perl's pack w" ); @nums = map { $_ * 2**31 } 0 .. 2000; $_ += int( rand( 2**16 ) ) for @nums; -check_round_trip( 'c64', \@nums ); +check_round_trip( 'cu64', \@nums ); # rand (always?) has 64-bit precision, but we need 32-bit - so truncate via # pack/unpack. @@ -152,6 +152,6 @@ my $unibytes = $latin; utf8ify($unibytes); utf8_flag_off($unibytes); my $slurped = $ram_file->get_contents; -substr( $slurped, 0, 1, "" ); # ditch c32 at head of string; +substr( $slurped, 0, 1, "" ); # ditch cu32 at head of string; is( $slurped, $unibytes, "write_string upgrades to utf8" ); diff --git a/perl/xs/Lucy/Document/Doc.c b/perl/xs/Lucy/Document/Doc.c index 902df23bd..a923e25c6 100644 --- a/perl/xs/Lucy/Document/Doc.c +++ b/perl/xs/Lucy/Document/Doc.c @@ -92,11 +92,11 @@ void LUCY_Doc_Serialize_IMP(lucy_Doc *self, lucy_OutStream *outstream) { dTHX; lucy_DocIVARS *const ivars = lucy_Doc_IVARS(self); - LUCY_OutStream_Write_C32(outstream, ivars->doc_id); + LUCY_OutStream_Write_CU32(outstream, ivars->doc_id); SV *frozen = S_nfreeze_fields(aTHX_ self); STRLEN len; char *buf = SvPV(frozen, len); - LUCY_OutStream_Write_C64(outstream, len); + LUCY_OutStream_Write_CU64(outstream, len); LUCY_OutStream_Write_Bytes(outstream, buf, len); SvREFCNT_dec(frozen); } @@ -104,7 +104,7 @@ LUCY_Doc_Serialize_IMP(lucy_Doc *self, lucy_OutStream *outstream) { static HV* S_thaw_fields(pTHX_ lucy_InStream *instream) { // Read frozen data into an SV buffer. - size_t len = (size_t)LUCY_InStream_Read_C64(instream); + size_t len = (size_t)LUCY_InStream_Read_CU64(instream); SV *buf_sv = newSV(len + 1); SvPOK_on(buf_sv); SvCUR_set(buf_sv, len); @@ -137,7 +137,7 @@ S_thaw_fields(pTHX_ lucy_InStream *instream) { lucy_Doc* LUCY_Doc_Deserialize_IMP(lucy_Doc *self, lucy_InStream *instream) { dTHX; - int32_t doc_id = (int32_t)LUCY_InStream_Read_C32(instream); + int32_t doc_id = (int32_t)LUCY_InStream_Read_CU32(instream); HV *fields = S_thaw_fields(aTHX_ instream); lucy_Doc_init(self, fields, doc_id); SvREFCNT_dec(fields); diff --git a/perl/xs/Lucy/Index/DocReader.c b/perl/xs/Lucy/Index/DocReader.c index 8eda8bc2c..7daf909d6 100644 --- a/perl/xs/Lucy/Index/DocReader.c +++ b/perl/xs/Lucy/Index/DocReader.c @@ -43,7 +43,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) { LUCY_InStream_Seek(ix_in, (int64_t)doc_id * 8); start = LUCY_InStream_Read_U64(ix_in); LUCY_InStream_Seek(dat_in, start); - num_fields = LUCY_InStream_Read_C32(dat_in); + num_fields = LUCY_InStream_Read_CU32(dat_in); // Decode stored data and build up the doc field by field. while (num_fields--) { @@ -53,7 +53,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) { lucy_FieldType *type; // Read field name. - field_name_len = LUCY_InStream_Read_C32(dat_in); + field_name_len = LUCY_InStream_Read_CU32(dat_in); field_name_ptr = SvGROW(field_name_sv, field_name_len + 1); LUCY_InStream_Read_Bytes(dat_in, field_name_ptr, field_name_len); SvPOK_on(field_name_sv); @@ -69,7 +69,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) { // Read the field value. switch (LUCY_FType_Primitive_ID(type) & lucy_FType_PRIMITIVE_ID_MASK) { case lucy_FType_TEXT: { - STRLEN value_len = LUCY_InStream_Read_C32(dat_in); + STRLEN value_len = LUCY_InStream_Read_CU32(dat_in); value_sv = newSV((value_len ? value_len : 1)); LUCY_InStream_Read_Bytes(dat_in, SvPVX(value_sv), value_len); SvCUR_set(value_sv, value_len); @@ -79,7 +79,7 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) { break; } case lucy_FType_BLOB: { - STRLEN value_len = LUCY_InStream_Read_C32(dat_in); + STRLEN value_len = LUCY_InStream_Read_CU32(dat_in); value_sv = newSV((value_len ? value_len : 1)); LUCY_InStream_Read_Bytes(dat_in, SvPVX(value_sv), value_len); SvCUR_set(value_sv, value_len); @@ -94,15 +94,15 @@ LUCY_DefDocReader_Fetch_Doc_IMP(lucy_DefaultDocReader *self, int32_t doc_id) { value_sv = newSVnv(LUCY_InStream_Read_F64(dat_in)); break; case lucy_FType_INT32: - value_sv = newSViv((int32_t)LUCY_InStream_Read_C32(dat_in)); + value_sv = newSViv(LUCY_InStream_Read_CI32(dat_in)); break; case lucy_FType_INT64: if (sizeof(IV) == 8) { - int64_t val = (int64_t)LUCY_InStream_Read_C64(dat_in); + int64_t val = LUCY_InStream_Read_CI64(dat_in); value_sv = newSViv((IV)val); } else { // (lossy) - int64_t val = (int64_t)LUCY_InStream_Read_C64(dat_in); + int64_t val = LUCY_InStream_Read_CI64(dat_in); value_sv = newSVnv((double)val); } break; From 795626b6c4c6c98cd6f6ac0d0a2899e4b6e09e33 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Tue, 19 Apr 2016 19:19:28 -0700 Subject: [PATCH 08/15] Switch Go bindings from C32/C64 to CI32/etc. Change IO methods invoked to use explicitly signed/unsigned versions rather than casting. --- go/build.go | 8 ++++++ go/lucy/lucy.go | 16 +++++------ go/lucy/store.go | 66 ++++++++++++++++++++++++++++++++++++++++++- go/lucy/store_test.go | 28 ++++++++++++------ 4 files changed, 101 insertions(+), 17 deletions(-) diff --git a/go/build.go b/go/build.go index 8353d66c3..ecf36f443 100644 --- a/go/build.go +++ b/go/build.go @@ -388,7 +388,11 @@ func specClasses(parcel *cfc.Parcel) { inStreamBinding.SpecMethod("Read_U32", "ReadU32() (uint32, error)") inStreamBinding.SpecMethod("Read_U64", "ReadU64() (uint64, error)") inStreamBinding.SpecMethod("Read_C32", "ReadC32() (uint32, error)") + inStreamBinding.SpecMethod("Read_CI32", "ReadCI32() (int32, error)") + inStreamBinding.SpecMethod("Read_CU32", "ReadCU32() (uint32, error)") inStreamBinding.SpecMethod("Read_C64", "ReadC64() (uint64, error)") + inStreamBinding.SpecMethod("Read_CI64", "ReadCI64() (int64, error)") + inStreamBinding.SpecMethod("Read_CU64", "ReadCU64() (uint64, error)") inStreamBinding.SpecMethod("Read_F32", "ReadF32() (float32, error)") inStreamBinding.SpecMethod("Read_F64", "ReadF64() (float64, error)") inStreamBinding.Register() @@ -406,7 +410,11 @@ func specClasses(parcel *cfc.Parcel) { outStreamBinding.SpecMethod("Write_U32", "WriteU32(uint32) error") outStreamBinding.SpecMethod("Write_U64", "WriteU64(uint64) error") outStreamBinding.SpecMethod("Write_C32", "WriteC32(uint32) error") + outStreamBinding.SpecMethod("Write_CI32", "WriteCI32(int32) error") + outStreamBinding.SpecMethod("Write_CU32", "WriteCU32(uint32) error") outStreamBinding.SpecMethod("Write_C64", "WriteC64(uint64) error") + outStreamBinding.SpecMethod("Write_CI64", "WriteCI64(int64) error") + outStreamBinding.SpecMethod("Write_CU64", "WriteCU64(uint64) error") outStreamBinding.SpecMethod("Write_F32", "WriteF32(float32) error") outStreamBinding.SpecMethod("Write_F64", "WriteF64(float64) error") outStreamBinding.SpecMethod("Absorb", "Absorb(InStream) error") diff --git a/go/lucy/lucy.go b/go/lucy/lucy.go index 138b599c8..4a4fc714b 100644 --- a/go/lucy/lucy.go +++ b/go/lucy/lucy.go @@ -297,7 +297,7 @@ func GOLUCY_Doc_Serialize(d *C.lucy_Doc, outstream *C.lucy_OutStream) { hash := clownfish.GoToClownfish(fields, unsafe.Pointer(C.CFISH_HASH), false) defer C.cfish_decref(hash) C.lucy_Freezer_serialize_hash((*C.cfish_Hash)(hash), outstream) - C.LUCY_OutStream_Write_C32(outstream, C.uint32_t(ivars.doc_id)) + C.LUCY_OutStream_Write_CI32(outstream, ivars.doc_id) } //export GOLUCY_Doc_Deserialize @@ -308,7 +308,7 @@ func GOLUCY_Doc_Deserialize(d *C.lucy_Doc, instream *C.lucy_InStream) *C.lucy_Do fields := clownfish.ToGo(hash) fieldsID := registry.store(fields) ivars.fields = unsafe.Pointer(fieldsID) - ivars.doc_id = C.int32_t(C.LUCY_InStream_Read_C32(instream)) + ivars.doc_id = C.LUCY_InStream_Read_CI32(instream) return d } @@ -473,12 +473,12 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{}) C.LUCY_InStream_Seek(ixInstream, C.int64_t(docID*8)) start := C.LUCY_InStream_Read_U64(ixInstream) C.LUCY_InStream_Seek(datInstream, C.int64_t(start)) - numFields := uint32(C.LUCY_InStream_Read_C32(datInstream)) + numFields := uint32(C.LUCY_InStream_Read_CU32(datInstream)) // Decode stored data and build up the doc field by field. for i := uint32(0); i < numFields; i++ { // Read field name. - fieldNameLen := C.size_t(C.LUCY_InStream_Read_C32(datInstream)) + fieldNameLen := C.size_t(C.LUCY_InStream_Read_CU32(datInstream)) if fieldNameLen > fieldNameCap { fieldNameCap = fieldNameLen fieldName = ((*C.char)(C.realloc(unsafe.Pointer(fieldName), fieldNameCap+1))) @@ -497,7 +497,7 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{}) // Read the field value. switch C.LUCY_FType_Primitive_ID(fieldType) & C.lucy_FType_PRIMITIVE_ID_MASK { case C.lucy_FType_TEXT: - valueLen := C.size_t(C.LUCY_InStream_Read_C32(datInstream)) + valueLen := C.size_t(C.LUCY_InStream_Read_CU32(datInstream)) buf := ((*C.char)(C.malloc(valueLen + 1))) C.LUCY_InStream_Read_Bytes(datInstream, buf, valueLen) val := C.GoStringN(buf, C.int(valueLen)) @@ -506,7 +506,7 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{}) return err } case C.lucy_FType_BLOB: - valueLen := C.size_t(C.LUCY_InStream_Read_C32(datInstream)) + valueLen := C.size_t(C.LUCY_InStream_Read_CU32(datInstream)) buf := ((*C.char)(C.malloc(valueLen))) C.LUCY_InStream_Read_Bytes(datInstream, buf, valueLen) val := C.GoBytes(unsafe.Pointer(buf), C.int(valueLen)) @@ -525,12 +525,12 @@ func doReadDocData(ddrC *C.lucy_DefaultDocReader, docID int32, doc interface{}) return err } case C.lucy_FType_INT32: - err := setField(fields, fieldNameGo, int32(C.LUCY_InStream_Read_C32(datInstream))) + err := setField(fields, fieldNameGo, int32(C.LUCY_InStream_Read_CI32(datInstream))) if err != nil { return err } case C.lucy_FType_INT64: - err := setField(fields, fieldNameGo, int64(C.LUCY_InStream_Read_C64(datInstream))) + err := setField(fields, fieldNameGo, int64(C.LUCY_InStream_Read_CI64(datInstream))) if err != nil { return err } diff --git a/go/lucy/store.go b/go/lucy/store.go index 7af82297e..0b11a2fe1 100644 --- a/go/lucy/store.go +++ b/go/lucy/store.go @@ -95,7 +95,7 @@ func (in *InStreamIMP) ReadString() (string, error) { var retval string err := clownfish.TrapErr(func() { self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) - size := C.size_t(C.LUCY_InStream_Read_C32(self)) + size := C.size_t(C.LUCY_InStream_Read_CU32(self)) buf := (*C.char)(C.malloc(size)) defer C.free(unsafe.Pointer(buf)) C.LUCY_InStream_Read_Bytes(self, buf, size) @@ -180,6 +180,24 @@ func (in *InStreamIMP) ReadC32() (uint32, error) { return retval, err } +func (in *InStreamIMP) ReadCI32() (int32, error) { + var retval int32 + err := clownfish.TrapErr(func() { + self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) + retval = int32(C.LUCY_InStream_Read_CI32(self)) + }) + return retval, err +} + +func (in *InStreamIMP) ReadCU32() (uint32, error) { + var retval uint32 + err := clownfish.TrapErr(func() { + self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) + retval = uint32(C.LUCY_InStream_Read_CU32(self)) + }) + return retval, err +} + func (in *InStreamIMP) ReadC64() (uint64, error) { var retval uint64 err := clownfish.TrapErr(func() { @@ -189,6 +207,24 @@ func (in *InStreamIMP) ReadC64() (uint64, error) { return retval, err } +func (in *InStreamIMP) ReadCI64() (int64, error) { + var retval int64 + err := clownfish.TrapErr(func() { + self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) + retval = int64(C.LUCY_InStream_Read_CI64(self)) + }) + return retval, err +} + +func (in *InStreamIMP) ReadCU64() (uint64, error) { + var retval uint64 + err := clownfish.TrapErr(func() { + self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) + retval = uint64(C.LUCY_InStream_Read_CU64(self)) + }) + return retval, err +} + func (in *InStreamIMP) ReadF32() (float32, error) { var retval float32 err := clownfish.TrapErr(func() { @@ -314,6 +350,20 @@ func (out *OutStreamIMP) WriteC32(value uint32) error { }) } +func (out *OutStreamIMP) WriteCI32(value int32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) + C.LUCY_OutStream_Write_CI32(self, C.int32_t(value)) + }) +} + +func (out *OutStreamIMP) WriteCU32(value uint32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) + C.LUCY_OutStream_Write_CU32(self, C.uint32_t(value)) + }) +} + func (out *OutStreamIMP) WriteC64(value uint64) error { return clownfish.TrapErr(func() { self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) @@ -321,6 +371,20 @@ func (out *OutStreamIMP) WriteC64(value uint64) error { }) } +func (out *OutStreamIMP) WriteCI64(value int64) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) + C.LUCY_OutStream_Write_CI64(self, C.int64_t(value)) + }) +} + +func (out *OutStreamIMP) WriteCU64(value uint64) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) + C.LUCY_OutStream_Write_CU64(self, C.uint64_t(value)) + }) +} + func (out *OutStreamIMP) WriteF32(value float32) error { return clownfish.TrapErr(func() { self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) diff --git a/go/lucy/store_test.go b/go/lucy/store_test.go index c4225256d..d87d539cf 100644 --- a/go/lucy/store_test.go +++ b/go/lucy/store_test.go @@ -215,11 +215,17 @@ func TestIOStreamReadWrite(t *testing.T) { if err = outStream.WriteU64(42); err != nil { t.Errorf("WriteU64: %s", err) } - if err = outStream.WriteC32(42); err != nil { - t.Errorf("WriteC32: %s", err) + if err = outStream.WriteCI32(42); err != nil { + t.Errorf("WriteCI32: %s", err) } - if err = outStream.WriteC64(42); err != nil { - t.Errorf("WriteC64: %s", err) + if err = outStream.WriteCU32(42); err != nil { + t.Errorf("WriteCU32: %s", err) + } + if err = outStream.WriteCI64(42); err != nil { + t.Errorf("WriteCI64: %s", err) + } + if err = outStream.WriteCU64(42); err != nil { + t.Errorf("WriteCU64: %s", err) } if err = outStream.WriteF32(1.5); err != nil { t.Errorf("WriteF32: %s", err) @@ -265,11 +271,17 @@ func TestIOStreamReadWrite(t *testing.T) { if got, err := inStream.ReadU64(); got != 42 || err != nil { t.Errorf("ReadU64: %d, %s", got, err) } - if got, err := inStream.ReadC32(); got != 42 || err != nil { - t.Errorf("ReadC32: %d, %s", got, err) + if got, err := inStream.ReadCI32(); got != 42 || err != nil { + t.Errorf("ReadCI32: %d, %s", got, err) + } + if got, err := inStream.ReadCU32(); got != 42 || err != nil { + t.Errorf("ReadCU32: %d, %s", got, err) + } + if got, err := inStream.ReadCI64(); got != 42 || err != nil { + t.Errorf("ReadCI64: %d, %s", got, err) } - if got, err := inStream.ReadC64(); got != 42 || err != nil { - t.Errorf("ReadC64: %d, %s", got, err) + if got, err := inStream.ReadCU64(); got != 42 || err != nil { + t.Errorf("ReadCU64: %d, %s", got, err) } if got, err := inStream.ReadF32(); got != 1.5 || err != nil { t.Errorf("ReadF32: %d, %s", got, err) From 17a3d60bbe2b5a1279052278225b2875668b2968 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 15:29:23 -0700 Subject: [PATCH 09/15] Change easy-to-review Read_C32 to CU32. All of these have the type of the lvalue show up in the diff, making it easy to confirm that the types match. --- c/src/Lucy/Index/DocReader.c | 8 ++++---- core/Lucy/Index/HighlightReader.c | 2 +- core/Lucy/Index/LexIndex.c | 4 ++-- core/Lucy/Index/Posting/MatchPosting.c | 6 +++--- core/Lucy/Index/Posting/RichPosting.c | 6 +++--- core/Lucy/Index/Posting/ScorePosting.c | 4 ++-- core/Lucy/Plan/TextType.c | 6 +++--- core/Lucy/Search/SortSpec.c | 2 +- core/Lucy/Util/Freezer.c | 12 ++++++------ core/LucyX/Search/ProximityQuery.c | 2 +- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c index 1e7767bac..3610dfc10 100644 --- a/c/src/Lucy/Index/DocReader.c +++ b/c/src/Lucy/Index/DocReader.c @@ -71,7 +71,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { // Read the field value. switch (FType_Primitive_ID(type) & FType_PRIMITIVE_ID_MASK) { case FType_TEXT: { - uint32_t value_len = InStream_Read_C32(dat_in); + uint32_t value_len = InStream_Read_CU32(dat_in); char *buf = (char*)MALLOCATE(value_len + 1); InStream_Read_Bytes(dat_in, buf, value_len); buf[value_len] = '\0'; @@ -79,7 +79,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { break; } case FType_BLOB: { - uint32_t value_len = InStream_Read_C32(dat_in); + uint32_t value_len = InStream_Read_CU32(dat_in); char *buf = (char*)MALLOCATE(value_len); InStream_Read_Bytes(dat_in, buf, value_len); value = (Obj*)Blob_new_steal(buf, value_len); @@ -92,10 +92,10 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { value = (Obj*)Float_new(InStream_Read_F64(dat_in)); break; case FType_INT32: - value = (Obj*)Int_new((int32_t)InStream_Read_C32(dat_in)); + value = (Obj*)Int_new(InStream_Read_CI32(dat_in)); break; case FType_INT64: - value = (Obj*)Int_new((int64_t)InStream_Read_C64(dat_in)); + value = (Obj*)Int_new(InStream_Read_CI64(dat_in)); break; default: value = NULL; diff --git a/core/Lucy/Index/HighlightReader.c b/core/Lucy/Index/HighlightReader.c index 2afaf66ed..590a6bdc1 100644 --- a/core/Lucy/Index/HighlightReader.c +++ b/core/Lucy/Index/HighlightReader.c @@ -200,7 +200,7 @@ DefHLReader_Fetch_Doc_Vec_IMP(DefaultHighlightReader *self, int32_t doc_id) { int64_t file_pos = InStream_Read_I64(ix_in); InStream_Seek(dat_in, file_pos); - uint32_t num_fields = InStream_Read_C32(dat_in); + uint32_t num_fields = InStream_Read_CU32(dat_in); while (num_fields--) { String *field = Freezer_read_string(dat_in); Blob *field_buf = Freezer_read_blob(dat_in); diff --git a/core/Lucy/Index/LexIndex.c b/core/Lucy/Index/LexIndex.c index 162933f12..2c6586b2e 100644 --- a/core/Lucy/Index/LexIndex.c +++ b/core/Lucy/Index/LexIndex.c @@ -129,11 +129,11 @@ S_read_entry(LexIndex *self) { int64_t offset = (int64_t)NumUtil_decode_bigend_u64(ivars->offsets + ivars->tick); InStream_Seek(ix_in, offset); TermStepper_Read_Key_Frame(ivars->term_stepper, ix_in); - int32_t doc_freq = InStream_Read_C32(ix_in); + int32_t doc_freq = InStream_Read_CI32(ix_in); TInfo_Set_Doc_Freq(tinfo, doc_freq); TInfo_Set_Post_FilePos(tinfo, InStream_Read_C64(ix_in)); int64_t skip_filepos = doc_freq >= ivars->skip_interval - ? InStream_Read_C64(ix_in) + ? InStream_Read_CI64(ix_in) : 0; TInfo_Set_Skip_FilePos(tinfo, skip_filepos); TInfo_Set_Lex_FilePos(tinfo, InStream_Read_C64(ix_in)); diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c index c0ead4eeb..ae4e1f22f 100644 --- a/core/Lucy/Index/Posting/MatchPosting.c +++ b/core/Lucy/Index/Posting/MatchPosting.c @@ -81,7 +81,7 @@ MatchPost_Reset_IMP(MatchPosting *self) { void MatchPost_Read_Record_IMP(MatchPosting *self, InStream *instream) { MatchPostingIVARS *const ivars = MatchPost_IVARS(self); - const uint32_t doc_code = InStream_Read_C32(instream); + const uint32_t doc_code = InStream_Read_CU32(instream); const uint32_t doc_delta = doc_code >> 1; // Apply delta doc and retrieve freq. @@ -100,12 +100,12 @@ MatchPost_Read_Raw_IMP(MatchPosting *self, InStream *instream, MemoryPool *mem_pool) { const char *const text_buf = Str_Get_Ptr8(term_text); const size_t text_size = Str_Get_Size(term_text); - const uint32_t doc_code = InStream_Read_C32(instream); + const uint32_t doc_code = InStream_Read_CU32(instream); const uint32_t delta_doc = doc_code >> 1; const int32_t doc_id = last_doc_id + delta_doc; const uint32_t freq = (doc_code & 1) ? 1 - : InStream_Read_C32(instream); + : InStream_Read_CU32(instream); const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING); size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size); void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes); diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c index b3ff7e42d..db590d41d 100644 --- a/core/Lucy/Index/Posting/RichPosting.c +++ b/core/Lucy/Index/Posting/RichPosting.c @@ -72,7 +72,7 @@ RichPost_Read_Record_IMP(RichPosting *self, InStream *instream) { float aggregate_weight = 0.0; // Decode delta doc. - uint32_t doc_code = InStream_Read_C32(instream); + uint32_t doc_code = InStream_Read_CU32(instream); ivars->doc_id += doc_code >> 1; // If the stored num was odd, the freq is 1. @@ -158,12 +158,12 @@ RichPost_Read_Raw_IMP(RichPosting *self, InStream *instream, MemoryPool *mem_pool) { const char *const text_buf = Str_Get_Ptr8(term_text); const size_t text_size = Str_Get_Size(term_text); - const uint32_t doc_code = InStream_Read_C32(instream); + const uint32_t doc_code = InStream_Read_CU32(instream); const uint32_t delta_doc = doc_code >> 1; const int32_t doc_id = last_doc_id + delta_doc; const uint32_t freq = (doc_code & 1) ? 1 - : InStream_Read_C32(instream); + : InStream_Read_CU32(instream); const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING); size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size, freq); void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes); diff --git a/core/Lucy/Index/Posting/ScorePosting.c b/core/Lucy/Index/Posting/ScorePosting.c index bc84bf88a..048fa219f 100644 --- a/core/Lucy/Index/Posting/ScorePosting.c +++ b/core/Lucy/Index/Posting/ScorePosting.c @@ -177,12 +177,12 @@ ScorePost_Read_Raw_IMP(ScorePosting *self, InStream *instream, MemoryPool *mem_pool) { const char *const text_buf = Str_Get_Ptr8(term_text); const size_t text_size = Str_Get_Size(term_text); - const uint32_t doc_code = InStream_Read_C32(instream); + const uint32_t doc_code = InStream_Read_CU32(instream); const uint32_t delta_doc = doc_code >> 1; const int32_t doc_id = last_doc_id + delta_doc; const uint32_t freq = (doc_code & 1) ? 1 - : InStream_Read_C32(instream); + : InStream_Read_CU32(instream); const size_t base_size = Class_Get_Obj_Alloc_Size(RAWPOSTING); size_t raw_post_bytes = MAX_RAW_POSTING_LEN(base_size, text_size, freq); void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes); diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c index 4737b8cad..4b84fa8c5 100644 --- a/core/Lucy/Plan/TextType.c +++ b/core/Lucy/Plan/TextType.c @@ -151,7 +151,7 @@ void TextTermStepper_Read_Key_Frame_IMP(TextTermStepper *self, InStream *instream) { TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self); - const uint32_t text_len = InStream_Read_C32(instream); + const uint32_t text_len = InStream_Read_CU32(instream); // Allocate space. char *ptr = BB_Grow(ivars->bytebuf, text_len); @@ -173,8 +173,8 @@ TextTermStepper_Read_Key_Frame_IMP(TextTermStepper *self, void TextTermStepper_Read_Delta_IMP(TextTermStepper *self, InStream *instream) { TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self); - const uint32_t text_overlap = InStream_Read_C32(instream); - const uint32_t finish_chars_len = InStream_Read_C32(instream); + const uint32_t text_overlap = InStream_Read_CU32(instream); + const uint32_t finish_chars_len = InStream_Read_CU32(instream); const uint32_t total_text_len = text_overlap + finish_chars_len; // Allocate space. diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c index c6fa2e438..9994bb4b2 100644 --- a/core/Lucy/Search/SortSpec.c +++ b/core/Lucy/Search/SortSpec.c @@ -53,7 +53,7 @@ SortSpec_Destroy_IMP(SortSpec *self) { SortSpec* SortSpec_Deserialize_IMP(SortSpec *self, InStream *instream) { - uint32_t num_rules = InStream_Read_C32(instream); + uint32_t num_rules = InStream_Read_CU32(instream); Vector *rules = Vec_new(num_rules); // Add rules. diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c index fa0d121b8..20e5e33b0 100644 --- a/core/Lucy/Util/Freezer.c +++ b/core/Lucy/Util/Freezer.c @@ -126,7 +126,7 @@ Freezer_deserialize(Obj *obj, InStream *instream) { obj = (Obj*)Freezer_deserialize_hash((Hash*)obj, instream); } else if (Obj_is_a(obj, INTEGER)) { - int64_t value = (int64_t)InStream_Read_C64(instream); + int64_t value = InStream_Read_CI64(instream); obj = (Obj*)Int_init((Integer*)obj, value); } else if (Obj_is_a(obj, FLOAT)) { @@ -245,11 +245,11 @@ Freezer_serialize_varray(Vector *array, OutStream *outstream) { Vector* Freezer_deserialize_varray(Vector *array, InStream *instream) { - uint32_t size = InStream_Read_C32(instream); + uint32_t size = InStream_Read_CU32(instream); Vec_init(array, size); - for (uint32_t tick = InStream_Read_C32(instream); + for (uint32_t tick = InStream_Read_CU32(instream); tick < size; - tick += InStream_Read_C32(instream) + tick += InStream_Read_CU32(instream) ) { Obj *obj = THAW(instream); Vec_Store(array, tick, obj); @@ -281,12 +281,12 @@ Freezer_serialize_hash(Hash *hash, OutStream *outstream) { Hash* Freezer_deserialize_hash(Hash *hash, InStream *instream) { - uint32_t size = InStream_Read_C32(instream); + uint32_t size = InStream_Read_CU32(instream); Hash_init(hash, size); while (size--) { - uint32_t len = InStream_Read_C32(instream); + uint32_t len = InStream_Read_CU32(instream); char *key_buf = (char*)MALLOCATE(len + 1); InStream_Read_Bytes(instream, key_buf, len); key_buf[len] = '\0'; diff --git a/core/LucyX/Search/ProximityQuery.c b/core/LucyX/Search/ProximityQuery.c index 6b01d5be2..637fcce67 100644 --- a/core/LucyX/Search/ProximityQuery.c +++ b/core/LucyX/Search/ProximityQuery.c @@ -95,7 +95,7 @@ ProximityQuery_Deserialize_IMP(ProximityQuery *self, InStream *instream) { float boost = InStream_Read_F32(instream); String *field = Freezer_read_string(instream); Vector *terms = Freezer_read_varray(instream); - uint32_t within = InStream_Read_C32(instream); + uint32_t within = InStream_Read_CU32(instream); return S_do_init(self, field, terms, boost, within); } From 5e4dcd2e31a80ed20e86ca931bbbc565f6781820 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 18:16:06 -0700 Subject: [PATCH 10/15] Change size_t C32/C64 read/writes. Adapt Read_C32/Write_C32/Read_C64/Write_C64 to Read_CU32/etc for variables representing memory sizes: String, Blob, Vector and Hash sizes in particular. --- c/src/Lucy/Index/DocReader.c | 6 +++--- core/Lucy/Index/DocWriter.c | 12 ++++++++++-- core/Lucy/Index/TermVector.c | 4 ++-- core/Lucy/Plan/TextType.c | 3 ++- core/Lucy/Search/SortSpec.c | 3 ++- core/Lucy/Util/Freezer.c | 22 +++++++++++++++------- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c index 3610dfc10..16e184e16 100644 --- a/c/src/Lucy/Index/DocReader.c +++ b/c/src/Lucy/Index/DocReader.c @@ -40,7 +40,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { Hash *const fields = Hash_new(1); int64_t start; uint32_t num_fields; - uint32_t field_name_cap = 31; + size_t field_name_cap = 31; char *field_name = (char*)MALLOCATE(field_name_cap + 1); // Get data file pointer from index, read number of fields. @@ -51,12 +51,12 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { // Decode stored data and build up the doc field by field. while (num_fields--) { - uint32_t field_name_len; + size_t field_name_len; Obj *value; FieldType *type; // Read field name. - field_name_len = InStream_Read_C32(dat_in); + field_name_len = InStream_Read_CU32(dat_in); if (field_name_len > field_name_cap) { field_name_cap = field_name_len; field_name = (char*)REALLOCATE(field_name, diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c index 818f45bee..ec583ccc8 100644 --- a/core/Lucy/Index/DocWriter.c +++ b/core/Lucy/Index/DocWriter.c @@ -119,14 +119,22 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter, case FType_TEXT: { const char *buf = Str_Get_Ptr8((String*)value); size_t size = Str_Get_Size((String*)value); - OutStream_Write_C32(dat_out, size); + if (size > INT32_MAX) { + THROW(ERR, "Field %o over 2GB: %u64", field, + (uint64_t)size); + } + OutStream_Write_CU32(dat_out, (uint32_t)size); OutStream_Write_Bytes(dat_out, buf, size); break; } case FType_BLOB: { const char *buf = Blob_Get_Buf((Blob*)value); size_t size = Blob_Get_Size((Blob*)value); - OutStream_Write_C32(dat_out, size); + if (size > INT32_MAX) { + THROW(ERR, "Field %o over 2GB: %u64", field, + (uint64_t)size); + } + OutStream_Write_CU32(dat_out, (uint32_t)size); OutStream_Write_Bytes(dat_out, buf, size); break; } diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c index b1a3391c1..9c3a6f99d 100644 --- a/core/Lucy/Index/TermVector.c +++ b/core/Lucy/Index/TermVector.c @@ -90,7 +90,7 @@ TV_Serialize_IMP(TermVector *self, OutStream *target) { Freezer_serialize_string(ivars->field, target); Freezer_serialize_string(ivars->text, target); - OutStream_Write_C64(target, ivars->num_pos); + OutStream_Write_CU64(target, ivars->num_pos); for (size_t i = 0; i < ivars->num_pos; i++) { OutStream_Write_C32(target, posits[i]); @@ -103,7 +103,7 @@ TermVector* TV_Deserialize_IMP(TermVector *self, InStream *instream) { String *field = Freezer_read_string(instream); String *text = Freezer_read_string(instream); - size_t num_pos = InStream_Read_C64(instream); + size_t num_pos = InStream_Read_CU64(instream); // Read positional data. int32_t *posits = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c index 4b84fa8c5..67f64a349 100644 --- a/core/Lucy/Plan/TextType.c +++ b/core/Lucy/Plan/TextType.c @@ -101,7 +101,8 @@ TextTermStepper_Write_Key_Frame_IMP(TextTermStepper *self, String *string = (String*)CERTIFY(value, STRING); const char *buf = Str_Get_Ptr8(string); size_t size = Str_Get_Size(string); - OutStream_Write_C32(outstream, size); + // Skip size check because we're deep in the internals. + OutStream_Write_CU32(outstream, (uint32_t)size); OutStream_Write_Bytes(outstream, buf, size); S_set_value(self, value); diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c index 9994bb4b2..21a6370cc 100644 --- a/core/Lucy/Search/SortSpec.c +++ b/core/Lucy/Search/SortSpec.c @@ -75,8 +75,9 @@ SortSpec_Get_Rules_IMP(SortSpec *self) { void SortSpec_Serialize_IMP(SortSpec *self, OutStream *target) { SortSpecIVARS *const ivars = SortSpec_IVARS(self); + // Skip size check. uint32_t num_rules = (uint32_t)Vec_Get_Size(ivars->rules); - OutStream_Write_C32(target, num_rules); + OutStream_Write_CU32(target, num_rules); for (uint32_t i = 0; i < num_rules; i++) { SortRule *rule = (SortRule*)Vec_Fetch(ivars->rules, i); SortRule_Serialize(rule, target); diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c index 20e5e33b0..90a63c393 100644 --- a/core/Lucy/Util/Freezer.c +++ b/core/Lucy/Util/Freezer.c @@ -180,13 +180,16 @@ void Freezer_serialize_string(String *string, OutStream *outstream) { size_t size = Str_Get_Size(string); const char *buf = Str_Get_Ptr8(string); - OutStream_Write_C64(outstream, size); + if (size > INT32_MAX) { + THROW(ERR, "Can't serialize string above 2GB: %u64", (uint64_t)size); + } + OutStream_Write_CU64(outstream, size); OutStream_Write_Bytes(outstream, buf, size); } String* Freezer_deserialize_string(String *string, InStream *instream) { - size_t size = InStream_Read_C32(instream); + size_t size = InStream_Read_CU32(instream); if (size == SIZE_MAX) { THROW(ERR, "Can't deserialize SIZE_MAX bytes"); } @@ -208,13 +211,16 @@ Freezer_read_string(InStream *instream) { void Freezer_serialize_blob(Blob *blob, OutStream *outstream) { size_t size = Blob_Get_Size(blob); - OutStream_Write_C32(outstream, size); + if (size > INT32_MAX) { + THROW(ERR, "Can't serialize blob above 2GB: %u64", (uint64_t)size); + } + OutStream_Write_CU64(outstream, size); OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size); } Blob* Freezer_deserialize_blob(Blob *blob, InStream *instream) { - size_t size = InStream_Read_C32(instream); + size_t size = InStream_Read_CU32(instream); char *buf = (char*)MALLOCATE(size); InStream_Read_Bytes(instream, buf, size); return Blob_init_steal(blob, buf, size); @@ -229,8 +235,9 @@ Freezer_read_blob(InStream *instream) { void Freezer_serialize_varray(Vector *array, OutStream *outstream) { uint32_t last_valid_tick = 0; + // Skip size check. uint32_t size = (uint32_t)Vec_Get_Size(array); - OutStream_Write_C32(outstream, size); + OutStream_Write_CU32(outstream, size); for (uint32_t i = 0; i < size; i++) { Obj *elem = Vec_Fetch(array, i); if (elem) { @@ -266,8 +273,9 @@ Freezer_read_varray(InStream *instream) { void Freezer_serialize_hash(Hash *hash, OutStream *outstream) { - uint32_t hash_size = Hash_Get_Size(hash); - OutStream_Write_C32(outstream, hash_size); + // Skip size check. + uint32_t hash_size = (uint32_t)Hash_Get_Size(hash); + OutStream_Write_CU32(outstream, hash_size); HashIterator *iter = HashIter_new(hash); while (HashIter_Next(iter)) { From e5ba8a19dd8e04039e7de5479b852150b7705b46 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 18:34:30 -0700 Subject: [PATCH 11/15] Switch C64 to CI64 for file positions. Change Read/Write C64 to CI64 for file positions, all of which are represented as signed 64-bit integers. --- core/Lucy/Index/LexIndex.c | 4 ++-- core/Lucy/Index/LexiconWriter.c | 2 +- core/Lucy/Index/Posting/MatchPosting.c | 16 ++++++++-------- core/Lucy/Index/SkipStepper.c | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/Lucy/Index/LexIndex.c b/core/Lucy/Index/LexIndex.c index 2c6586b2e..8a406a345 100644 --- a/core/Lucy/Index/LexIndex.c +++ b/core/Lucy/Index/LexIndex.c @@ -131,12 +131,12 @@ S_read_entry(LexIndex *self) { TermStepper_Read_Key_Frame(ivars->term_stepper, ix_in); int32_t doc_freq = InStream_Read_CI32(ix_in); TInfo_Set_Doc_Freq(tinfo, doc_freq); - TInfo_Set_Post_FilePos(tinfo, InStream_Read_C64(ix_in)); + TInfo_Set_Post_FilePos(tinfo, InStream_Read_CI64(ix_in)); int64_t skip_filepos = doc_freq >= ivars->skip_interval ? InStream_Read_CI64(ix_in) : 0; TInfo_Set_Skip_FilePos(tinfo, skip_filepos); - TInfo_Set_Lex_FilePos(tinfo, InStream_Read_C64(ix_in)); + TInfo_Set_Lex_FilePos(tinfo, InStream_Read_CI64(ix_in)); } void diff --git a/core/Lucy/Index/LexiconWriter.c b/core/Lucy/Index/LexiconWriter.c index 1ef32f10a..7c996d5a0 100644 --- a/core/Lucy/Index/LexiconWriter.c +++ b/core/Lucy/Index/LexiconWriter.c @@ -100,7 +100,7 @@ S_add_last_term_to_ix(LexiconWriter *self) { ivars->ix_out, TermStepper_Get_Value(ivars->term_stepper)); TermStepper_Write_Key_Frame(ivars->tinfo_stepper, ivars->ix_out, TermStepper_Get_Value(ivars->tinfo_stepper)); - OutStream_Write_C64(ivars->ix_out, OutStream_Tell(ivars->dat_out)); + OutStream_Write_CI64(ivars->ix_out, OutStream_Tell(ivars->dat_out)); ivars->ix_count++; } diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c index ae4e1f22f..c7348fafa 100644 --- a/core/Lucy/Index/Posting/MatchPosting.c +++ b/core/Lucy/Index/Posting/MatchPosting.c @@ -275,11 +275,11 @@ MatchTInfoStepper_Write_Key_Frame_IMP(MatchTermInfoStepper *self, OutStream_Write_C32(outstream, doc_freq); // Write postings file pointer. - OutStream_Write_C64(outstream, tinfo_ivars->post_filepos); + OutStream_Write_CI64(outstream, tinfo_ivars->post_filepos); // Write skip file pointer (maybe). if (doc_freq >= ivars->skip_interval) { - OutStream_Write_C64(outstream, tinfo_ivars->skip_filepos); + OutStream_Write_CI64(outstream, tinfo_ivars->skip_filepos); } TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo); @@ -299,11 +299,11 @@ MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self, OutStream_Write_C32(outstream, doc_freq); // Write postings file pointer delta. - OutStream_Write_C64(outstream, post_delta); + OutStream_Write_CI64(outstream, post_delta); // Write skip file pointer (maybe). if (doc_freq >= ivars->skip_interval) { - OutStream_Write_C64(outstream, TInfo_IVARS(tinfo)->skip_filepos); + OutStream_Write_CI64(outstream, TInfo_IVARS(tinfo)->skip_filepos); } TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo); @@ -319,11 +319,11 @@ MatchTInfoStepper_Read_Key_Frame_IMP(MatchTermInfoStepper *self, tinfo_ivars->doc_freq = InStream_Read_C32(instream); // Read postings file pointer. - tinfo_ivars->post_filepos = InStream_Read_C64(instream); + tinfo_ivars->post_filepos = InStream_Read_CI64(instream); // Maybe read skip pointer. if (tinfo_ivars->doc_freq >= ivars->skip_interval) { - tinfo_ivars->skip_filepos = InStream_Read_C64(instream); + tinfo_ivars->skip_filepos = InStream_Read_CI64(instream); } else { tinfo_ivars->skip_filepos = 0; @@ -339,11 +339,11 @@ MatchTInfoStepper_Read_Delta_IMP(MatchTermInfoStepper *self, InStream *instream) tinfo_ivars->doc_freq = InStream_Read_C32(instream); // Adjust postings file pointer. - tinfo_ivars->post_filepos += InStream_Read_C64(instream); + tinfo_ivars->post_filepos += InStream_Read_CI64(instream); // Maybe read skip pointer. if (tinfo_ivars->doc_freq >= ivars->skip_interval) { - tinfo_ivars->skip_filepos = InStream_Read_C64(instream); + tinfo_ivars->skip_filepos = InStream_Read_CI64(instream); } else { tinfo_ivars->skip_filepos = 0; diff --git a/core/Lucy/Index/SkipStepper.c b/core/Lucy/Index/SkipStepper.c index 9f22dc9a8..634e2d861 100644 --- a/core/Lucy/Index/SkipStepper.c +++ b/core/Lucy/Index/SkipStepper.c @@ -48,7 +48,7 @@ void SkipStepper_Read_Record_IMP(SkipStepper *self, InStream *instream) { SkipStepperIVARS *const ivars = SkipStepper_IVARS(self); ivars->doc_id += InStream_Read_C32(instream); - ivars->filepos += InStream_Read_C64(instream); + ivars->filepos += InStream_Read_CI64(instream); } String* @@ -69,7 +69,7 @@ SkipStepper_Write_Record_IMP(SkipStepper *self, OutStream *outstream, OutStream_Write_C32(outstream, delta_doc_id); // Write delta file pointer. - OutStream_Write_C64(outstream, delta_filepos); + OutStream_Write_CI64(outstream, delta_filepos); } From 3089bfdf4c09b7cf3f937405417c93e23063d727 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 18:40:26 -0700 Subject: [PATCH 12/15] Switch C32 code relating to doc IDs. Doc IDs are signed. Delta doc IDs may signed or unsigned. Use the appropriate CI32 or CU32 read/write methods for both. --- c/src/Lucy/Document/Doc.c | 4 ++-- core/Lucy/Index/Posting/MatchPosting.c | 4 ++-- core/Lucy/Index/Posting/RawPosting.c | 4 ++-- core/Lucy/Index/SkipStepper.c | 4 ++-- core/Lucy/Search/MatchDoc.c | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/c/src/Lucy/Document/Doc.c b/c/src/Lucy/Document/Doc.c index d0fb065cb..19b9d9dbd 100644 --- a/c/src/Lucy/Document/Doc.c +++ b/c/src/Lucy/Document/Doc.c @@ -69,14 +69,14 @@ Doc_Serialize_IMP(Doc *self, OutStream *outstream) { DocIVARS *const ivars = Doc_IVARS(self); Hash *hash = (Hash*)ivars->fields; Freezer_serialize_hash(hash, outstream); - OutStream_Write_C32(outstream, ivars->doc_id); + OutStream_Write_CI32(outstream, ivars->doc_id); } Doc* Doc_Deserialize_IMP(Doc *self, InStream *instream) { DocIVARS *const ivars = Doc_IVARS(self); ivars->fields = Freezer_read_hash(instream); - ivars->doc_id = InStream_Read_C32(instream); + ivars->doc_id = InStream_Read_CI32(instream); return self; } diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c index c7348fafa..2bda909fc 100644 --- a/core/Lucy/Index/Posting/MatchPosting.c +++ b/core/Lucy/Index/Posting/MatchPosting.c @@ -212,11 +212,11 @@ MatchPostWriter_Write_Posting_IMP(MatchPostingWriter *self, RawPosting *posting) + posting_ivars->content_len; if (posting_ivars->freq == 1) { const uint32_t doc_code = (delta_doc << 1) | 1; - OutStream_Write_C32(outstream, doc_code); + OutStream_Write_CU32(outstream, doc_code); } else { const uint32_t doc_code = delta_doc << 1; - OutStream_Write_C32(outstream, doc_code); + OutStream_Write_CU32(outstream, doc_code); OutStream_Write_C32(outstream, posting_ivars->freq); } OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len); diff --git a/core/Lucy/Index/Posting/RawPosting.c b/core/Lucy/Index/Posting/RawPosting.c index 06e5367c3..052020ad5 100644 --- a/core/Lucy/Index/Posting/RawPosting.c +++ b/core/Lucy/Index/Posting/RawPosting.c @@ -133,11 +133,11 @@ RawPostWriter_Write_Posting_IMP(RawPostingWriter *self, RawPosting *posting) { + posting_ivars->content_len; if (posting_ivars->freq == 1) { const uint32_t doc_code = (delta_doc << 1) | 1; - OutStream_Write_C32(outstream, doc_code); + OutStream_Write_CU32(outstream, doc_code); } else { const uint32_t doc_code = delta_doc << 1; - OutStream_Write_C32(outstream, doc_code); + OutStream_Write_CU32(outstream, doc_code); OutStream_Write_C32(outstream, posting_ivars->freq); } OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len); diff --git a/core/Lucy/Index/SkipStepper.c b/core/Lucy/Index/SkipStepper.c index 634e2d861..2fe5b3c76 100644 --- a/core/Lucy/Index/SkipStepper.c +++ b/core/Lucy/Index/SkipStepper.c @@ -47,7 +47,7 @@ SkipStepper_Set_ID_And_Filepos_IMP(SkipStepper *self, int32_t doc_id, void SkipStepper_Read_Record_IMP(SkipStepper *self, InStream *instream) { SkipStepperIVARS *const ivars = SkipStepper_IVARS(self); - ivars->doc_id += InStream_Read_C32(instream); + ivars->doc_id += InStream_Read_CI32(instream); ivars->filepos += InStream_Read_CI64(instream); } @@ -66,7 +66,7 @@ SkipStepper_Write_Record_IMP(SkipStepper *self, OutStream *outstream, const int64_t delta_filepos = ivars->filepos - last_filepos; // Write delta doc id. - OutStream_Write_C32(outstream, delta_doc_id); + OutStream_Write_CI32(outstream, delta_doc_id); // Write delta file pointer. OutStream_Write_CI64(outstream, delta_filepos); diff --git a/core/Lucy/Search/MatchDoc.c b/core/Lucy/Search/MatchDoc.c index bbc4b5937..1ac57edd8 100644 --- a/core/Lucy/Search/MatchDoc.c +++ b/core/Lucy/Search/MatchDoc.c @@ -47,7 +47,7 @@ MatchDoc_Destroy_IMP(MatchDoc *self) { void MatchDoc_Serialize_IMP(MatchDoc *self, OutStream *outstream) { MatchDocIVARS *const ivars = MatchDoc_IVARS(self); - OutStream_Write_C32(outstream, ivars->doc_id); + OutStream_Write_CI32(outstream, ivars->doc_id); OutStream_Write_F32(outstream, ivars->score); OutStream_Write_U8(outstream, ivars->values ? 1 : 0); if (ivars->values) { Freezer_serialize_varray(ivars->values, outstream); } @@ -56,7 +56,7 @@ MatchDoc_Serialize_IMP(MatchDoc *self, OutStream *outstream) { MatchDoc* MatchDoc_Deserialize_IMP(MatchDoc *self, InStream *instream) { MatchDocIVARS *const ivars = MatchDoc_IVARS(self); - ivars->doc_id = InStream_Read_C32(instream); + ivars->doc_id = InStream_Read_CI32(instream); ivars->score = InStream_Read_F32(instream); if (InStream_Read_U8(instream)) { ivars->values = Freezer_read_varray(instream); From 656a773b89d3676dc9d62d48f5a2d145342bb32c Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 18:52:32 -0700 Subject: [PATCH 13/15] Change C32/C64 for freq, positions. `freq` (num occurrences per document for a term) is generally represented as a uint32_t. Positions may be signed or unsigned -- it's not consistent. Just match up to existing usage. --- core/Lucy/Index/Posting/MatchPosting.c | 6 +++--- core/Lucy/Index/Posting/RawPosting.c | 2 +- core/Lucy/Index/Posting/RichPosting.c | 4 ++-- core/Lucy/Index/TermVector.c | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c index 2bda909fc..e44eee86e 100644 --- a/core/Lucy/Index/Posting/MatchPosting.c +++ b/core/Lucy/Index/Posting/MatchPosting.c @@ -90,7 +90,7 @@ MatchPost_Read_Record_IMP(MatchPosting *self, InStream *instream) { ivars->freq = 1; } else { - ivars->freq = InStream_Read_C32(instream); + ivars->freq = InStream_Read_CU32(instream); } } @@ -217,7 +217,7 @@ MatchPostWriter_Write_Posting_IMP(MatchPostingWriter *self, RawPosting *posting) else { const uint32_t doc_code = delta_doc << 1; OutStream_Write_CU32(outstream, doc_code); - OutStream_Write_C32(outstream, posting_ivars->freq); + OutStream_Write_CU32(outstream, posting_ivars->freq); } OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len); ivars->last_doc_id = doc_id; @@ -296,7 +296,7 @@ MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self, - TInfo_IVARS(last_tinfo)->post_filepos; // Write doc_freq. - OutStream_Write_C32(outstream, doc_freq); + OutStream_Write_CU32(outstream, doc_freq); // Write postings file pointer delta. OutStream_Write_CI64(outstream, post_delta); diff --git a/core/Lucy/Index/Posting/RawPosting.c b/core/Lucy/Index/Posting/RawPosting.c index 052020ad5..0fc5b2e8e 100644 --- a/core/Lucy/Index/Posting/RawPosting.c +++ b/core/Lucy/Index/Posting/RawPosting.c @@ -138,7 +138,7 @@ RawPostWriter_Write_Posting_IMP(RawPostingWriter *self, RawPosting *posting) { else { const uint32_t doc_code = delta_doc << 1; OutStream_Write_CU32(outstream, doc_code); - OutStream_Write_C32(outstream, posting_ivars->freq); + OutStream_Write_CU32(outstream, posting_ivars->freq); } OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len); ivars->last_doc_id = doc_id; diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c index db590d41d..da9da772d 100644 --- a/core/Lucy/Index/Posting/RichPosting.c +++ b/core/Lucy/Index/Posting/RichPosting.c @@ -81,7 +81,7 @@ RichPost_Read_Record_IMP(RichPosting *self, InStream *instream) { } // Otherwise, freq was stored as a C32. else { - ivars->freq = InStream_Read_C32(instream); + ivars->freq = InStream_Read_CU32(instream); } // Read positions, aggregate per-position boost byte into weight. @@ -96,7 +96,7 @@ RichPost_Read_Record_IMP(RichPosting *self, InStream *instream) { float *prox_boosts = ivars->prox_boosts; while (num_prox--) { - position += InStream_Read_C32(instream); + position += InStream_Read_CU32(instream); *positions++ = position; *prox_boosts = norm_decoder[InStream_Read_U8(instream)]; aggregate_weight += *prox_boosts; diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c index 9c3a6f99d..8ad0bcf3e 100644 --- a/core/Lucy/Index/TermVector.c +++ b/core/Lucy/Index/TermVector.c @@ -93,9 +93,9 @@ TV_Serialize_IMP(TermVector *self, OutStream *target) { OutStream_Write_CU64(target, ivars->num_pos); for (size_t i = 0; i < ivars->num_pos; i++) { - OutStream_Write_C32(target, posits[i]); - OutStream_Write_C32(target, starts[i]); - OutStream_Write_C32(target, ends[i]); + OutStream_Write_CI32(target, posits[i]); + OutStream_Write_CI32(target, starts[i]); + OutStream_Write_CI32(target, ends[i]); } } @@ -110,9 +110,9 @@ TV_Deserialize_IMP(TermVector *self, InStream *instream) { int32_t *starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); int32_t *ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); for (size_t i = 0; i < num_pos; i++) { - posits[i] = InStream_Read_C32(instream); - starts[i] = InStream_Read_C32(instream); - ends[i] = InStream_Read_C32(instream); + posits[i] = InStream_Read_CI32(instream); + starts[i] = InStream_Read_CI32(instream); + ends[i] = InStream_Read_CI32(instream); } I32Array *positions = I32Arr_new_steal(posits, num_pos); I32Array *start_offsets = I32Arr_new_steal(starts, num_pos); From febd2ff3cb76b460e0778e6d732830cf1f860e50 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 19:01:15 -0700 Subject: [PATCH 14/15] Misc C32/C64 transition. Transition all remaining Read/Write C32/C64 to Read_CI64/etc. --- c/src/Lucy/Index/DocReader.c | 2 +- core/Lucy/Index/DocWriter.c | 6 +++--- core/Lucy/Index/HighlightWriter.c | 2 +- core/Lucy/Index/Posting/MatchPosting.c | 6 +++--- core/Lucy/Plan/TextType.c | 2 +- core/Lucy/Search/SortRule.c | 8 ++++---- core/Lucy/Search/TopDocs.c | 4 ++-- core/Lucy/Util/Freezer.c | 6 +++--- core/LucyX/Search/ProximityQuery.c | 6 +++--- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c index 16e184e16..97c8df042 100644 --- a/c/src/Lucy/Index/DocReader.c +++ b/c/src/Lucy/Index/DocReader.c @@ -47,7 +47,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { InStream_Seek(ix_in, (int64_t)doc_id * 8); start = InStream_Read_U64(ix_in); InStream_Seek(dat_in, start); - num_fields = InStream_Read_C32(dat_in); + num_fields = InStream_Read_CU32(dat_in); // Decode stored data and build up the doc field by field. while (num_fields--) { diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c index ec583ccc8..43ac738d0 100644 --- a/core/Lucy/Index/DocWriter.c +++ b/core/Lucy/Index/DocWriter.c @@ -105,7 +105,7 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter, FieldType *type = Inverter_Get_Type(inverter); if (FType_Stored(type)) { num_stored++; } } - OutStream_Write_C32(dat_out, num_stored); + OutStream_Write_CU32(dat_out, num_stored); Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { @@ -140,12 +140,12 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter, } case FType_INT32: { int32_t val = (int32_t)Int_Get_Value((Integer*)value); - OutStream_Write_C32(dat_out, val); + OutStream_Write_CI32(dat_out, val); break; } case FType_INT64: { int64_t val = Int_Get_Value((Integer*)value); - OutStream_Write_C64(dat_out, val); + OutStream_Write_CI64(dat_out, val); break; } case FType_FLOAT32: { diff --git a/core/Lucy/Index/HighlightWriter.c b/core/Lucy/Index/HighlightWriter.c index d81b623db..72153b24d 100644 --- a/core/Lucy/Index/HighlightWriter.c +++ b/core/Lucy/Index/HighlightWriter.c @@ -121,7 +121,7 @@ HLWriter_Add_Inverted_Doc_IMP(HighlightWriter *self, Inverter *inverter, num_highlightable++; } } - OutStream_Write_C32(dat_out, num_highlightable); + OutStream_Write_CU32(dat_out, num_highlightable); Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c index e44eee86e..cfe9efdc8 100644 --- a/core/Lucy/Index/Posting/MatchPosting.c +++ b/core/Lucy/Index/Posting/MatchPosting.c @@ -272,7 +272,7 @@ MatchTInfoStepper_Write_Key_Frame_IMP(MatchTermInfoStepper *self, TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)value); // Write doc_freq. - OutStream_Write_C32(outstream, doc_freq); + OutStream_Write_CI32(outstream, doc_freq); // Write postings file pointer. OutStream_Write_CI64(outstream, tinfo_ivars->post_filepos); @@ -316,7 +316,7 @@ MatchTInfoStepper_Read_Key_Frame_IMP(MatchTermInfoStepper *self, TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)ivars->value); // Read doc freq. - tinfo_ivars->doc_freq = InStream_Read_C32(instream); + tinfo_ivars->doc_freq = InStream_Read_CI32(instream); // Read postings file pointer. tinfo_ivars->post_filepos = InStream_Read_CI64(instream); @@ -336,7 +336,7 @@ MatchTInfoStepper_Read_Delta_IMP(MatchTermInfoStepper *self, InStream *instream) TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)ivars->value); // Read doc freq. - tinfo_ivars->doc_freq = InStream_Read_C32(instream); + tinfo_ivars->doc_freq = InStream_Read_CI32(instream); // Adjust postings file pointer. tinfo_ivars->post_filepos += InStream_Read_CI64(instream); diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c index 67f64a349..318fa47d2 100644 --- a/core/Lucy/Plan/TextType.c +++ b/core/Lucy/Plan/TextType.c @@ -135,7 +135,7 @@ TextTermStepper_Write_Delta_IMP(TextTermStepper *self, OutStream *outstream, const size_t diff_len = new_size - overlap; // Write number of common bytes and common bytes. - OutStream_Write_C32(outstream, overlap); + OutStream_Write_CI32(outstream, overlap); OutStream_Write_String(outstream, diff_start_str, diff_len); // Update value. diff --git a/core/Lucy/Search/SortRule.c b/core/Lucy/Search/SortRule.c index 736205c6c..dfb650c1f 100644 --- a/core/Lucy/Search/SortRule.c +++ b/core/Lucy/Search/SortRule.c @@ -63,22 +63,22 @@ SortRule_Destroy_IMP(SortRule *self) { SortRule* SortRule_Deserialize_IMP(SortRule *self, InStream *instream) { SortRuleIVARS *ivars = SortRule_IVARS(self); - ivars->type = InStream_Read_C32(instream); + ivars->type = InStream_Read_CI32(instream); if (ivars->type == SortRule_FIELD) { ivars->field = Freezer_read_string(instream); } - ivars->reverse = !!InStream_Read_C32(instream); + ivars->reverse = !!InStream_Read_CU32(instream); return self; } void SortRule_Serialize_IMP(SortRule *self, OutStream *target) { SortRuleIVARS *ivars = SortRule_IVARS(self); - OutStream_Write_C32(target, ivars->type); + OutStream_Write_CI32(target, ivars->type); if (ivars->type == SortRule_FIELD) { Freezer_serialize_string(ivars->field, target); } - OutStream_Write_C32(target, !!ivars->reverse); + OutStream_Write_CU32(target, !!ivars->reverse); } String* diff --git a/core/Lucy/Search/TopDocs.c b/core/Lucy/Search/TopDocs.c index ddaeb13b6..b68504803 100644 --- a/core/Lucy/Search/TopDocs.c +++ b/core/Lucy/Search/TopDocs.c @@ -51,14 +51,14 @@ void TopDocs_Serialize_IMP(TopDocs *self, OutStream *outstream) { TopDocsIVARS *const ivars = TopDocs_IVARS(self); Freezer_serialize_varray(ivars->match_docs, outstream); - OutStream_Write_C32(outstream, ivars->total_hits); + OutStream_Write_CU32(outstream, ivars->total_hits); } TopDocs* TopDocs_Deserialize_IMP(TopDocs *self, InStream *instream) { TopDocsIVARS *const ivars = TopDocs_IVARS(self); ivars->match_docs = Freezer_read_varray(instream); - ivars->total_hits = InStream_Read_C32(instream); + ivars->total_hits = InStream_Read_CU32(instream); return self; } diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c index 90a63c393..26039d23e 100644 --- a/core/Lucy/Util/Freezer.c +++ b/core/Lucy/Util/Freezer.c @@ -68,7 +68,7 @@ Freezer_serialize(Obj *obj, OutStream *outstream) { } else if (Obj_is_a(obj, INTEGER)) { int64_t val = Int_Get_Value((Integer*)obj); - OutStream_Write_C64(outstream, (uint64_t)val); + OutStream_Write_CI64(outstream, val); } else if (Obj_is_a(obj, FLOAT)) { double val = Float_Get_Value((Float*)obj); @@ -241,13 +241,13 @@ Freezer_serialize_varray(Vector *array, OutStream *outstream) { for (uint32_t i = 0; i < size; i++) { Obj *elem = Vec_Fetch(array, i); if (elem) { - OutStream_Write_C32(outstream, i - last_valid_tick); + OutStream_Write_CU32(outstream, i - last_valid_tick); FREEZE(elem, outstream); last_valid_tick = i; } } // Terminate. - OutStream_Write_C32(outstream, size - last_valid_tick); + OutStream_Write_CU32(outstream, size - last_valid_tick); } Vector* diff --git a/core/LucyX/Search/ProximityQuery.c b/core/LucyX/Search/ProximityQuery.c index 637fcce67..1a9358475 100644 --- a/core/LucyX/Search/ProximityQuery.c +++ b/core/LucyX/Search/ProximityQuery.c @@ -87,7 +87,7 @@ ProximityQuery_Serialize_IMP(ProximityQuery *self, OutStream *outstream) { OutStream_Write_F32(outstream, ivars->boost); Freezer_serialize_string(ivars->field, outstream); Freezer_serialize_varray(ivars->terms, outstream); - OutStream_Write_C32(outstream, ivars->within); + OutStream_Write_CU32(outstream, ivars->within); } ProximityQuery* @@ -264,7 +264,7 @@ ProximityCompiler_Serialize_IMP(ProximityCompiler *self, OutStream_Write_F32(outstream, ivars->raw_weight); OutStream_Write_F32(outstream, ivars->query_norm_factor); OutStream_Write_F32(outstream, ivars->normalized_weight); - OutStream_Write_C32(outstream, ivars->within); + OutStream_Write_CU32(outstream, ivars->within); } ProximityCompiler* @@ -278,7 +278,7 @@ ProximityCompiler_Deserialize_IMP(ProximityCompiler *self, ivars->raw_weight = InStream_Read_F32(instream); ivars->query_norm_factor = InStream_Read_F32(instream); ivars->normalized_weight = InStream_Read_F32(instream); - ivars->within = InStream_Read_C32(instream); + ivars->within = InStream_Read_CU32(instream); return self; } From ab5a44bd414e79ed0a5127e303887056ce09d7cd Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Wed, 20 Apr 2016 19:23:29 -0700 Subject: [PATCH 15/15] Remove obsolete C32/C64 code. --- core/Lucy/Store/InStream.c | 10 ---------- core/Lucy/Store/InStream.cfh | 11 ----------- core/Lucy/Store/OutStream.c | 10 ---------- core/Lucy/Store/OutStream.cfh | 10 ---------- go/build.go | 4 ---- go/lucy/store.go | 32 -------------------------------- 6 files changed, 77 deletions(-) diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c index d04652d94..ab78b0dae 100644 --- a/core/Lucy/Store/InStream.c +++ b/core/Lucy/Store/InStream.c @@ -472,11 +472,6 @@ InStream_Read_F64_IMP(InStream *self) { return duo.d; } -uint32_t -InStream_Read_C32_IMP(InStream *self) { - return SI_read_cu32(self); -} - int32_t InStream_Read_CI32_IMP(InStream *self) { return (int32_t)SI_read_cu32(self); @@ -501,11 +496,6 @@ SI_read_cu32(InStream *self) { return retval; } -uint64_t -InStream_Read_C64_IMP(InStream *self) { - return SI_read_cu64(self); -} - int64_t InStream_Read_CI64_IMP(InStream *self) { return (int64_t)SI_read_cu64(self); diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh index 921be7765..3d9686046 100644 --- a/core/Lucy/Store/InStream.cfh +++ b/core/Lucy/Store/InStream.cfh @@ -171,11 +171,6 @@ class Lucy::Store::InStream inherits Clownfish::Obj { final double Read_F64(InStream *self); - /** Read in a compressed 32-bit unsigned integer. - */ - uint32_t - Read_C32(InStream *self); - /** Read in a compressed 32-bit signed integer. */ int32_t @@ -186,12 +181,6 @@ class Lucy::Store::InStream inherits Clownfish::Obj { uint32_t Read_CU32(InStream *self); - /** Read a 64-bit integer, using the same encoding as a C32 but occupying - * as many as 10 bytes. - */ - final uint64_t - Read_C64(InStream *self); - /** Read a 64-bit signed integer, using the same encoding as a CI32 but * occupying as many as 10 bytes. */ diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c index 8ae07d42f..7e619a444 100644 --- a/core/Lucy/Store/OutStream.c +++ b/core/Lucy/Store/OutStream.c @@ -289,11 +289,6 @@ OutStream_Write_F64_IMP(OutStream *self, double value) { SI_write_bytes(self, ivars, buf_copy, sizeof(double)); } -void -OutStream_Write_C32_IMP(OutStream *self, uint32_t value) { - SI_write_cu32(self, OutStream_IVARS(self), value); -} - void OutStream_Write_CI32_IMP(OutStream *self, int32_t value) { SI_write_cu32(self, OutStream_IVARS(self), (uint32_t)value); @@ -322,11 +317,6 @@ SI_write_cu32(OutStream *self, OutStreamIVARS *ivars, uint32_t value) { SI_write_bytes(self, ivars, ptr, (buf + sizeof(buf)) - ptr); } -void -OutStream_Write_C64_IMP(OutStream *self, uint64_t value) { - SI_write_cu64(self, OutStream_IVARS(self), value); -} - void OutStream_Write_CI64_IMP(OutStream *self, int64_t value) { SI_write_cu64(self, OutStream_IVARS(self), (uint64_t)value); diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh index cb6700369..391418d51 100644 --- a/core/Lucy/Store/OutStream.cfh +++ b/core/Lucy/Store/OutStream.cfh @@ -113,11 +113,6 @@ class Lucy::Store::OutStream inherits Clownfish::Obj { final void Write_U64(OutStream *self, uint64_t value); - /** Write a 32-bit integer using a compressed format. - */ - final void - Write_C32(OutStream *self, uint32_t value); - /** Write a signed 32-bit integer using a compressed format. */ final void @@ -128,11 +123,6 @@ class Lucy::Store::OutStream inherits Clownfish::Obj { final void Write_CU32(OutStream *self, uint32_t value); - /** Write a 64-bit integer using a compressed format. - */ - final void - Write_C64(OutStream *self, uint64_t value); - /** Write a signed 64-bit integer using a compressed format. */ final void diff --git a/go/build.go b/go/build.go index ecf36f443..3fea7a0d7 100644 --- a/go/build.go +++ b/go/build.go @@ -387,10 +387,8 @@ func specClasses(parcel *cfc.Parcel) { inStreamBinding.SpecMethod("Read_U8", "ReadU8() (uint8, error)") inStreamBinding.SpecMethod("Read_U32", "ReadU32() (uint32, error)") inStreamBinding.SpecMethod("Read_U64", "ReadU64() (uint64, error)") - inStreamBinding.SpecMethod("Read_C32", "ReadC32() (uint32, error)") inStreamBinding.SpecMethod("Read_CI32", "ReadCI32() (int32, error)") inStreamBinding.SpecMethod("Read_CU32", "ReadCU32() (uint32, error)") - inStreamBinding.SpecMethod("Read_C64", "ReadC64() (uint64, error)") inStreamBinding.SpecMethod("Read_CI64", "ReadCI64() (int64, error)") inStreamBinding.SpecMethod("Read_CU64", "ReadCU64() (uint64, error)") inStreamBinding.SpecMethod("Read_F32", "ReadF32() (float32, error)") @@ -409,10 +407,8 @@ func specClasses(parcel *cfc.Parcel) { outStreamBinding.SpecMethod("Write_U8", "WriteU8(uint8) error") outStreamBinding.SpecMethod("Write_U32", "WriteU32(uint32) error") outStreamBinding.SpecMethod("Write_U64", "WriteU64(uint64) error") - outStreamBinding.SpecMethod("Write_C32", "WriteC32(uint32) error") outStreamBinding.SpecMethod("Write_CI32", "WriteCI32(int32) error") outStreamBinding.SpecMethod("Write_CU32", "WriteCU32(uint32) error") - outStreamBinding.SpecMethod("Write_C64", "WriteC64(uint64) error") outStreamBinding.SpecMethod("Write_CI64", "WriteCI64(int64) error") outStreamBinding.SpecMethod("Write_CU64", "WriteCU64(uint64) error") outStreamBinding.SpecMethod("Write_F32", "WriteF32(float32) error") diff --git a/go/lucy/store.go b/go/lucy/store.go index 0b11a2fe1..a543ece41 100644 --- a/go/lucy/store.go +++ b/go/lucy/store.go @@ -171,15 +171,6 @@ func (in *InStreamIMP) ReadU64() (uint64, error) { return retval, err } -func (in *InStreamIMP) ReadC32() (uint32, error) { - var retval uint32 - err := clownfish.TrapErr(func() { - self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) - retval = uint32(C.LUCY_InStream_Read_C32(self)) - }) - return retval, err -} - func (in *InStreamIMP) ReadCI32() (int32, error) { var retval int32 err := clownfish.TrapErr(func() { @@ -198,15 +189,6 @@ func (in *InStreamIMP) ReadCU32() (uint32, error) { return retval, err } -func (in *InStreamIMP) ReadC64() (uint64, error) { - var retval uint64 - err := clownfish.TrapErr(func() { - self := (*C.lucy_InStream)(clownfish.Unwrap(in, "in")) - retval = uint64(C.LUCY_InStream_Read_C64(self)) - }) - return retval, err -} - func (in *InStreamIMP) ReadCI64() (int64, error) { var retval int64 err := clownfish.TrapErr(func() { @@ -343,13 +325,6 @@ func (out *OutStreamIMP) WriteU64(value uint64) error { }) } -func (out *OutStreamIMP) WriteC32(value uint32) error { - return clownfish.TrapErr(func() { - self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) - C.LUCY_OutStream_Write_C32(self, C.uint32_t(value)) - }) -} - func (out *OutStreamIMP) WriteCI32(value int32) error { return clownfish.TrapErr(func() { self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) @@ -364,13 +339,6 @@ func (out *OutStreamIMP) WriteCU32(value uint32) error { }) } -func (out *OutStreamIMP) WriteC64(value uint64) error { - return clownfish.TrapErr(func() { - self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out")) - C.LUCY_OutStream_Write_C64(self, C.uint64_t(value)) - }) -} - func (out *OutStreamIMP) WriteCI64(value int64) error { return clownfish.TrapErr(func() { self := (*C.lucy_OutStream)(clownfish.Unwrap(out, "out"))