Add more tests for compression

Unit tests for different data sequences, and SQL test for float4.

tsl/src/compression/deltadelta.c

@@ -155,8 +155,6 @@ deltadelta_compressor_append_null_value(Compressor *compressor)
 	delta_delta_compressor_append_null(extended->internal);
 }
 
-static void *delta_delta_compressor_finish(DeltaDeltaCompressor *compressor);
-
 static void *
 deltadelta_compressor_finish_and_reset(Compressor *compressor)
 {
@@ -331,7 +329,7 @@ delta_delta_from_parts(uint64 last_value, uint64 last_delta, Simple8bRleSerializ
 	return compressed;
 }
 
-static void *
+void *
 delta_delta_compressor_finish(DeltaDeltaCompressor *compressor)
 {
 	Simple8bRleSerialized *deltas = simple8brle_compressor_finish(&compressor->delta_delta);

tsl/src/compression/deltadelta.h

@@ -32,6 +32,7 @@ extern Compressor *delta_delta_compressor_for_type(Oid element_type);
 extern DeltaDeltaCompressor *delta_delta_compressor_alloc(void);
 extern void delta_delta_compressor_append_null(DeltaDeltaCompressor *compressor);
 extern void delta_delta_compressor_append_value(DeltaDeltaCompressor *compressor, int64 next_val);
+extern void *delta_delta_compressor_finish(DeltaDeltaCompressor *compressor);
 
 extern DecompressionIterator *
 delta_delta_decompression_iterator_from_datum_forward(Datum deltadelta_compressed,

tsl/src/compression/gorilla.c

@@ -262,13 +262,13 @@ gorilla_compressor_alloc(void)
 	return compressor;
 }
 
+/* This function is used for testing only. */
 Datum
 tsl_gorilla_compressor_append(PG_FUNCTION_ARGS)
 {
 	MemoryContext old_context;
 	MemoryContext agg_context;
-	GorillaCompressor *compressor =
-		(GorillaCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0));
+	Compressor *compressor = (Compressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0));
 
 	if (!AggCheckCallContext(fcinfo, &agg_context))
 	{
@@ -279,20 +279,38 @@ tsl_gorilla_compressor_append(PG_FUNCTION_ARGS)
 	old_context = MemoryContextSwitchTo(agg_context);
 
 	if (compressor == NULL)
-		compressor = gorilla_compressor_alloc();
+	{
+		compressor = gorilla_compressor_for_type(get_fn_expr_argtype(fcinfo->flinfo, 1));
+	}
 
 	if (PG_ARGISNULL(1))
-		gorilla_compressor_append_null(compressor);
+		compressor->append_null(compressor);
 	else
 	{
-		double next_val = PG_GETARG_FLOAT8(1);
-		gorilla_compressor_append_value(compressor, double_get_bits(next_val));
+		compressor->append_val(compressor, PG_GETARG_DATUM(1));
 	}
 
 	MemoryContextSwitchTo(old_context);
 	PG_RETURN_POINTER(compressor);
 }
 
+/* This function is used for testing only. */
+Datum
+tsl_gorilla_compressor_finish(PG_FUNCTION_ARGS)
+{
+	Compressor *compressor = (Compressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0));
+
+	if (compressor == NULL)
+		PG_RETURN_NULL();
+
+	void *compressed = compressor->finish(compressor);
+
+	if (compressed == NULL)
+		PG_RETURN_NULL();
+
+	PG_RETURN_POINTER(compressed);
+}
+
 void
 gorilla_compressor_append_null(GorillaCompressor *compressor)
 {
@@ -440,22 +458,6 @@ gorilla_compressor_finish(GorillaCompressor *compressor)
 	return compressed_gorilla_data_serialize(&data);
 }
 
-Datum
-tsl_gorilla_compressor_finish(PG_FUNCTION_ARGS)
-{
-	GorillaCompressor *compressor =
-		(GorillaCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0));
-	void *compressed;
-	if (compressor == NULL)
-		PG_RETURN_NULL();
-
-	compressed = gorilla_compressor_finish(compressor);
-	if (compressed == NULL)
-		PG_RETURN_NULL();
-
-	PG_RETURN_POINTER(compressed);
-}
-
 /*******************************
  ***  DecompressionIterator  ***
  *******************************/

tsl/test/sql/compression_algos.sql

@@ -9,14 +9,16 @@ AS :TSL_MODULE_PATHNAME LANGUAGE C VOLATILE;
 \ir include/compression_utils.sql
 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
 
+-- helper function: float -> pseudorandom float [0..1].
+create or replace function mix(x float4) returns float4 as $$ select ((hashfloat4(x) / (pow(2., 31) - 1) + 1) / 2)::float4 $$ language sql;
+create or replace function mix(x timestamptz) returns float4 as $$ select mix(extract(epoch from x)::float4) $$ language sql;
+
 ------------------
 -- C unit tests --
 ------------------
 
 SELECT ts_test_compression();
 
-\ir include/rand_generator.sql
-
 ------------------------
 -- BIGINT Compression --
 ------------------------
@@ -32,7 +34,7 @@ SELECT
 \set DECOMPRESS_REVERSE_CMD _timescaledb_internal.decompress_reverse(c::_timescaledb_internal.compressed_data, NULL::BIGINT)
 
 -- random order
-CREATE TABLE base_ints AS SELECT row_number() OVER() as rn, item::bigint FROM (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY gen_rand_minstd()) sub;
+CREATE TABLE base_ints AS SELECT row_number() OVER() as rn, item::bigint FROM (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY mix(item)) sub;
 \ir include/compression_test.sql
 DROP TABLE base_ints;
 
@@ -90,7 +92,7 @@ DROP TABLE base_ints;
 -- INT Compression --
 ------------------------
 
-CREATE TABLE base_ints AS SELECT row_number() OVER() as rn, item::int FROM (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY gen_rand_minstd()) sub;
+CREATE TABLE base_ints AS SELECT row_number() OVER() as rn, item::int FROM (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY mix(item)) sub;
 SELECT
   $$
   select item::bigint from base_ints order by rn
@@ -117,11 +119,74 @@ SELECT
 \set DECOMPRESS_REVERSE_CMD _timescaledb_internal.decompress_reverse(c::_timescaledb_internal.compressed_data, NULL::TIMESTAMPTZ)
 
 CREATE TABLE base_time AS SELECT row_number() OVER() as rn, item FROM
-    (select sub.item from (SELECT generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') item) as sub ORDER BY gen_rand_minstd()) sub;
+    (select sub.item from (SELECT generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') item) as sub ORDER BY mix(item)) sub;
 \ir include/compression_test.sql
 DROP TABLE base_time;
 
 
+------------------------
+-- FLOAT4 Compression --
+------------------------
+SELECT
+  $$
+  select item from base_floats order by rn
+  $$ AS "QUERY"
+\gset
+\set TABLE_NAME base_floats
+SELECT 'real' as "TYPE" \gset
+\set COMPRESSION_CMD _timescaledb_internal.compress_gorilla(item)
+SELECT '_timescaledb_internal.decompress_forward(c::_timescaledb_internal.compressed_data, NULL::float4)' AS "DECOMPRESS_FORWARD_CMD" \gset
+SELECT '_timescaledb_internal.decompress_reverse(c::_timescaledb_internal.compressed_data, NULL::float4)' AS "DECOMPRESS_REVERSE_CMD" \gset
+
+CREATE TABLE base_floats AS SELECT row_number() OVER() as rn, item::float4 FROM
+    (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY mix(item)) sub;
+\ir include/compression_test.sql
+SELECT c gorilla_text FROM compressed;
+DROP TABLE  base_floats;
+
+-- single element
+CREATE TABLE base_floats AS SELECT row_number() OVER() as rn, item::float4 FROM (SELECT generate_series(1, 1) item) sub;
+\ir include/compression_test.sql
+DROP TABLE base_floats;
+
+--special values
+CREATE TABLE base_floats AS SELECT row_number() over () as rn, item FROM
+    (
+        VALUES
+           --special
+           (0::float4), ('Infinity'), ('-Infinity'), ('NaN'),
+           --big deltas
+           (0), ('Infinity'), ('-Infinity'), ('Infinity'), ('-Infinity'),
+           (0), ('-Infinity'), (32), (5), ('-Infinity'), (-52), ('Infinity'),
+           (1000),
+           --big delta_deltas
+            (0), ('Infinity'), ('Infinity'), ('-Infinity'), ('-Infinity'), ('Infinity'), ('Infinity')
+    ) as t(item);
+\ir include/compression_test.sql
+DROP TABLE  base_floats;
+
+-- all 0s
+CREATE TABLE base_floats AS SELECT row_number() over () as rn, 0::float4 as item FROM (SELECT generate_series(1, 1000) ) j;
+\ir include/compression_test.sql
+DROP TABLE  base_floats;
+
+-- NULLs
+CREATE TABLE base_floats AS SELECT row_number() OVER() as rn, NULLIF(i, 5)::float4 item FROM generate_series(1, 10) i;
+\ir include/compression_test.sql
+DROP TABLE base_floats;
+
+CREATE TABLE base_floats AS SELECT row_number() OVER() as rn, NULLIF(i, 1)::float4 item FROM generate_series(1, 10) i;
+\ir include/compression_test.sql
+DROP TABLE base_floats;
+
+CREATE TABLE base_floats AS SELECT row_number() OVER() as rn, NULLIF(i, 10)::float4 item FROM generate_series(1, 10) i;
+\ir include/compression_test.sql
+DROP TABLE base_floats;
+
+CREATE TABLE base_floats AS SELECT row_number() OVER() as rn, NULLIF(NULLIF(NULLIF(NULLIF(i, 2), 4), 5), 8)::float4 item FROM generate_series(1, 10) i;
+\ir include/compression_test.sql
+DROP TABLE base_floats;
+
 ------------------------
 -- DOUBLE Compression --
 ------------------------
@@ -138,7 +203,7 @@ SELECT '_timescaledb_internal.decompress_forward(c::_timescaledb_internal.compre
 SELECT '_timescaledb_internal.decompress_reverse(c::_timescaledb_internal.compressed_data, NULL::DOUBLE PRECISION)' AS "DECOMPRESS_REVERSE_CMD" \gset
 
 CREATE TABLE base_doubles AS SELECT row_number() OVER() as rn, item::double precision FROM
-    (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY gen_rand_minstd()) sub;
+    (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY mix(item)) sub;
 \ir include/compression_test.sql
 SELECT c gorilla_text FROM compressed;
 DROP TABLE  base_doubles;
@@ -204,15 +269,15 @@ SELECT 'TEXT' as "TYPE" \gset
 
 -- high cardinality
 CREATE TABLE base_texts AS SELECT row_number() OVER() as rn, item::text FROM
-    (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY gen_rand_minstd()) sub;
+    (select sub.item from (SELECT generate_series(1, 1000) item) as sub ORDER BY mix(item)) sub;
 \ir include/compression_test.sql
 SELECT c from compressed;
 DROP TABLE base_texts;
 
 
 -- low cardinality
 CREATE TABLE base_texts AS SELECT row_number() OVER() as rn, item::text FROM
-    (SELECT i as item FROM generate_series(1, 10) i, generate_series(1, 100) j ORDER BY gen_rand_minstd()) sub;
+    (SELECT i as item FROM generate_series(1, 10) i, generate_series(1, 100) j ORDER BY mix(i + j)) sub;
 \ir include/compression_test.sql
 
 DROP TABLE base_texts;
@@ -224,7 +289,7 @@ DROP TABLE base_texts;
 
 -- high cardinality with toasted values
 CREATE TABLE base_texts AS SELECT row_number() OVER() as rn, repeat(item::text, 100000) as item FROM
-    (select sub.item from (SELECT generate_series(1, 10) item) as sub ORDER BY gen_rand_minstd()) sub;
+    (select sub.item from (SELECT generate_series(1, 10) item) as sub ORDER BY mix(item)) sub;
 --make sure it's toasted
 SELECT pg_total_relation_size(reltoastrelid)
       FROM pg_class c
@@ -267,7 +332,7 @@ SELECT 'TEXT' as "TYPE" \gset
 
 --basic test
 CREATE TABLE base_texts AS SELECT row_number() OVER() as rn, item::text FROM
-    (select sub.item from (SELECT generate_series(1, 100) item) as sub ORDER BY gen_rand_minstd()) sub;
+    (select sub.item from (SELECT generate_series(1, 100) item) as sub ORDER BY mix(item)) sub;
 \ir include/compression_test.sql
 SELECT c from compressed;
 DROP TABLE base_texts;
@@ -279,7 +344,7 @@ DROP TABLE base_texts;
 
 -- toasted values
 CREATE TABLE base_texts AS SELECT row_number() OVER() as rn, repeat(item::text, 100000) as item FROM
-    (select sub.item from (SELECT generate_series(1, 10) item) as sub ORDER BY gen_rand_minstd()) sub;
+    (select sub.item from (SELECT generate_series(1, 10) item) as sub ORDER BY mix(item)) sub;
 --make sure it's toasted
 SELECT pg_total_relation_size(reltoastrelid)
       FROM pg_class c

tsl/test/sql/include/compression_utils.sql

@@ -34,7 +34,7 @@ CREATE OR REPLACE FUNCTION _timescaledb_internal.deltadelta_compressor_finish(in
    AS :MODULE_PATHNAME, 'ts_deltadelta_compressor_finish'
    LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT;
 
-CREATE OR REPLACE FUNCTION _timescaledb_internal.gorilla_compressor_append(internal, DOUBLE PRECISION)
+CREATE OR REPLACE FUNCTION _timescaledb_internal.gorilla_compressor_append(internal, ANYELEMENT)
    RETURNS internal
    AS :MODULE_PATHNAME, 'ts_gorilla_compressor_append'
    LANGUAGE C IMMUTABLE PARALLEL SAFE;
@@ -86,7 +86,7 @@ CREATE AGGREGATE _timescaledb_internal.compress_deltadelta(timestamptz) (
     FINALFUNC = _timescaledb_internal.timestamptz_compress_finish
 );
 
-CREATE AGGREGATE _timescaledb_internal.compress_gorilla(DOUBLE PRECISION) (
+CREATE AGGREGATE _timescaledb_internal.compress_gorilla(ANYELEMENT) (
     STYPE = internal,
     SFUNC = _timescaledb_internal.gorilla_compressor_append,
     FINALFUNC = _timescaledb_internal.gorilla_compressor_finish