From f0372409d990b83d4216a8aecec8eeeed7eaf6d0 Mon Sep 17 00:00:00 2001 From: Murat Tuncer Date: Wed, 7 Jan 2015 16:18:42 +0200 Subject: [PATCH] reflected review changes for #44 --- Makefile | 3 +-- cstore_writer.c | 9 +++++---- expected/insert.out | 37 +++++++++++++++++++++++++++++++++++ expected/variable_length.out | 38 ------------------------------------ sql/insert.sql | 30 ++++++++++++++++++++++++++++ sql/variable_length.sql | 16 --------------- 6 files changed, 73 insertions(+), 60 deletions(-) delete mode 100644 expected/variable_length.out delete mode 100644 sql/variable_length.sql diff --git a/Makefile b/Makefile index d1d7491..1f1a828 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,7 @@ OBJS = cstore.pb-c.o cstore_fdw.o cstore_writer.o cstore_reader.o \ EXTENSION = cstore_fdw DATA = cstore_fdw--1.1.sql cstore_fdw--1.0--1.1.sql -REGRESS = create load query analyze data_types functions block_filtering drop \ - insert copyto variable_length +REGRESS = create load query analyze data_types functions block_filtering drop insert copyto EXTRA_CLEAN = cstore.pb-c.h cstore.pb-c.c data/*.cstore data/*.cstore.footer \ sql/block_filtering.sql sql/create.sql sql/data_types.sql sql/load.sql \ sql/copyto.sql expected/block_filtering.out expected/create.out \ diff --git a/cstore_writer.c b/cstore_writer.c index 7284fa9..d12b8e8 100644 --- a/cstore_writer.c +++ b/cstore_writer.c @@ -238,12 +238,13 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul int columnTypeLength = attributeForm->attlen; Oid columnCollation = attributeForm->attcollation; Datum columnValue = columnValues[columnIndex]; - bool variableLengthColumn = columnTypeLength == -1; - if (variableLengthColumn) + /* detoast variable length attributes if necessary */ + if (columnTypeLength == -1) { - columnValue = PointerGetDatum( - PG_DETOAST_DATUM(columnValues[columnIndex])); + struct varlena *detoastedValue = PG_DETOAST_DATUM(columnValue); + + columnValue = PointerGetDatum(detoastedValue); } blockData->existsArray[blockRowIndex] = true; diff --git a/expected/insert.out b/expected/insert.out index 8c29ff1..49d9ed1 100644 --- a/expected/insert.out +++ b/expected/insert.out @@ -49,3 +49,40 @@ select count(*) from test_insert_command; drop table test_insert_command_data; drop foreign table test_insert_command; +-- test long attribute value insertion +-- create sufficiently long text so that data is stored in toast +CREATE TABLE test_long_text AS +SELECT a as int_val, string_agg(random()::text, '') as text_val +FROM generate_series(1, 10) a, generate_series(1, 1000) b +GROUP BY a ORDER BY a; +-- store hash values of text for later comparison +CREATE TABLE test_long_text_hash AS +SELECT int_val, md5(text_val) AS hash +FROM test_long_text; +CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text) +SERVER cstore_server; +-- store long text in cstore table +INSERT INTO test_cstore_long_text SELECT * FROM test_long_text; +-- drop source table to remove original text from toast +DROP TABLE test_long_text; +-- check if text data is still available in cstore table +-- by comparing previously stored hash. +SELECT a.int_val +FROM test_long_text_hash a, test_cstore_long_text c +WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val); + int_val +--------- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +(10 rows) + +DROP TABLE test_long_text_hash; +DROP FOREIGN TABLE test_cstore_long_text; diff --git a/expected/variable_length.out b/expected/variable_length.out deleted file mode 100644 index 013d3d0..0000000 --- a/expected/variable_length.out +++ /dev/null @@ -1,38 +0,0 @@ -CREATE TABLE text_data AS -SELECT string_agg(random()::text, '') -FROM generate_series(1, 1000) a, generate_series(1, 10) b -GROUP BY b; -CREATE FOREIGN TABLE cstore_text_data(A text) SERVER cstore_server; -INSERT INTO cstore_text_data SELECT * FROM text_data; -SELECT char_length(md5(a)) FROM cstore_text_data; - char_length -------------- - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 -(10 rows) - -DROP TABLE text_data; -SELECT char_length(md5(a)) FROM cstore_text_data; - char_length -------------- - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 -(10 rows) - -DROP FOREIGN TABLE cstore_text_data; diff --git a/sql/insert.sql b/sql/insert.sql index 33966e5..7a6b075 100644 --- a/sql/insert.sql +++ b/sql/insert.sql @@ -24,3 +24,33 @@ select count(*) from test_insert_command; drop table test_insert_command_data; drop foreign table test_insert_command; + +-- test long attribute value insertion +-- create sufficiently long text so that data is stored in toast +CREATE TABLE test_long_text AS +SELECT a as int_val, string_agg(random()::text, '') as text_val +FROM generate_series(1, 10) a, generate_series(1, 1000) b +GROUP BY a ORDER BY a; + +-- store hash values of text for later comparison +CREATE TABLE test_long_text_hash AS +SELECT int_val, md5(text_val) AS hash +FROM test_long_text; + +CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text) +SERVER cstore_server; + +-- store long text in cstore table +INSERT INTO test_cstore_long_text SELECT * FROM test_long_text; + +-- drop source table to remove original text from toast +DROP TABLE test_long_text; + +-- check if text data is still available in cstore table +-- by comparing previously stored hash. +SELECT a.int_val +FROM test_long_text_hash a, test_cstore_long_text c +WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val); + +DROP TABLE test_long_text_hash; +DROP FOREIGN TABLE test_cstore_long_text; diff --git a/sql/variable_length.sql b/sql/variable_length.sql deleted file mode 100644 index 4c93107..0000000 --- a/sql/variable_length.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE text_data AS -SELECT string_agg(random()::text, '') -FROM generate_series(1, 1000) a, generate_series(1, 10) b -GROUP BY b; - -CREATE FOREIGN TABLE cstore_text_data(A text) SERVER cstore_server; - -INSERT INTO cstore_text_data SELECT * FROM text_data; - -SELECT char_length(md5(a)) FROM cstore_text_data; - -DROP TABLE text_data; - -SELECT char_length(md5(a)) FROM cstore_text_data; - -DROP FOREIGN TABLE cstore_text_data;