Skip to content

Commit

Permalink
Fixed batch look ahead in compressed sorted merge
Browse files Browse the repository at this point in the history 
In decompress_sorted_merge_get_next_tuple it is determine how many
batches need to be opened currently to perform a sorted merge. This is
done by checking if the first tuple from the last opened batch is larger
than the last returned tuple.

If a filter removes the first tuple, the first into the heap inserted
tuple from this batch can no longer be used to perform the check. This
patch fixes the wrong batch look ahead.

Fixes: timescale#5797
  • Loading branch information
@jnidzwetzki
jnidzwetzki committed Jun 26, 2023
1 parent da20d07 commit 33a3e10
Show file tree
Hide file tree
Showing 9 changed files with 712 additions and 6 deletions.
4 changes: 4 additions & 0 deletions .unreleased/bugfix_5798
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fixes: #5798 Fixed batch look ahead in compressed sorted merge

Thanks: @JamieD9 for reporting an issue with a wrong result ordering

9 changes: 6 additions & 3 deletions tsl/src/nodes/decompress_chunk/exec.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -926,10 +926,11 @@ decompress_chunk_explain(CustomScanState *node, List *ancestors, ExplainState *e
* in batch_state->decompressed_slot_projected. The slot will be empty if the batch
* is entirely processed.
*/
void
bool
decompress_get_next_tuple_from_batch(DecompressChunkState *chunk_state,
DecompressBatchState *batch_state)
{
bool first_tuple_returned = true;
TupleTableSlot *decompressed_slot_scan = batch_state->decompressed_slot_scan;
TupleTableSlot *decompressed_slot_projected = batch_state->decompressed_slot_projected;

Expand Down Expand Up @@ -962,7 +963,7 @@ decompress_get_next_tuple_from_batch(DecompressChunkState *chunk_state,
/* Clear old slot state */
ExecClearTuple(decompressed_slot_projected);

return;
return first_tuple_returned;
}

Assert(batch_state->initialized);
Expand Down Expand Up @@ -1027,9 +1028,11 @@ decompress_get_next_tuple_from_batch(DecompressChunkState *chunk_state,
if (is_valid_tuple)
{
Assert(!TTS_EMPTY(decompressed_slot_projected));
return;
return first_tuple_returned;
}

first_tuple_returned = false;

/* Otherwise fetch the next tuple in the next iteration */
}
}
Expand Down
2 changes: 1 addition & 1 deletion tsl/src/nodes/decompress_chunk/exec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ extern DecompressSlotNumber decompress_get_free_batch_state_id(DecompressChunkSt
extern void decompress_initialize_batch(DecompressChunkState *chunk_state,
DecompressBatchState *batch_state, TupleTableSlot *subslot);

extern void decompress_get_next_tuple_from_batch(DecompressChunkState *chunk_state,
extern bool decompress_get_next_tuple_from_batch(DecompressChunkState *chunk_state,
DecompressBatchState *batch_state);

extern void decompress_set_batch_state_to_unused(DecompressChunkState *chunk_state, int batch_id);
Expand Down
15 changes: 13 additions & 2 deletions tsl/src/nodes/decompress_chunk/sorted_merge.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ decompress_batch_open_next_batch(DecompressChunkState *chunk_state)

decompress_initialize_batch(chunk_state, batch_state, subslot);

decompress_get_next_tuple_from_batch(chunk_state, batch_state);
bool first_tuple_returned = decompress_get_next_tuple_from_batch(chunk_state, batch_state);

if (!TupIsNull(batch_state->decompressed_slot_projected))
{
Expand All @@ -118,7 +118,18 @@ decompress_batch_open_next_batch(DecompressChunkState *chunk_state)

chunk_state->most_recent_batch = batch_state_id;

return;
/* In decompress_sorted_merge_get_next_tuple it is determined how many batches
* need to be opened currently to perform a sorted merge. This is done by
* checking if the first tuple from the last opened batch is larger than the
* last returned tuple.
*
* If a filter removes the first tuple, the first into the heap inserted
* tuple from this batch can no longer be used to perform the check.
* Therefore, we must continue opening additional batches until the condition
* is met.
*/
if (first_tuple_returned)
return;
}
}
}
Expand Down
146 changes: 146 additions & 0 deletions tsl/test/expected/compression_sorted_merge-12.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1486,3 +1486,149 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign
Wed Jan 01 10:00:00 2020 | hin1111 | model111 | blok111 | message_here | signal1 | 12.34 | 12.34
(1 row)

-- Condition that filter the first tuple of a batch - Issue 5797
CREATE TABLE test (
id bigint,
dttm timestamp,
otherId int,
valueFk int,
otherFk int,
measure double precision
);
SELECT create_hypertable('test', 'dttm');
WARNING: column type "timestamp without time zone" used for "dttm" does not follow best practices
NOTICE: adding not-null constraint to column "dttm"
create_hypertable
--------------------
(13,public,test,t)
(1 row)

ALTER TABLE test SET (
timescaledb.compress,
timescaledb.compress_segmentby = 'otherId,valueFk,otherFk'
);
INSERT INTO public.test (id, dttm, otherid, valuefk, otherfk, measure)
VALUES (109288, '2023-05-25 23:12:13.000000', 130, 14499, 13, 0.13216569884001217),
(109286, '2023-05-25 23:12:13.000000', 130, 14500, 13, 0.3740651978942786),
(107617, '2023-05-25 14:24:12.000000', 130, 14850, 13, 0.5978259144311195),
(103864, '2023-05-25 13:30:51.000000', 130, 16760, 13, 0.4733429856616205),
(104977, '2023-05-25 12:11:47.000000', 133, 14843, 13, 0.24366893909655118),
(108321, '2023-05-25 18:39:07.000000', 133, 15294, 13, 0.8768629101819378),
(108320, '2023-05-25 16:09:17.000000', 133, 15294, 13, 0.6185638532799445),
(104987, '2023-05-25 13:27:19.000000', 133, 15294, 13, 0.9830846939109854),
(104737, '2023-05-25 19:59:54.000000', 135, 14238, 13, 0.2388520055224177),
(106278, '2023-05-25 19:59:54.000000', 135, 14238, 13, 0.6305156586688518),
(104741, '2023-05-25 19:59:54.000000', 135, 14238, 13, 0.4990673076480263),
(106277, '2023-05-25 12:53:34.000000', 135, 14238, 13, 0.46086278330000496),
(97409, '2023-05-25 12:38:48.000000', 137, 14533, 13, 0.8308173375978924),
(105234, '2023-05-25 12:38:45.000000', 137, 14533, 13, 0.10860962941223917),
(105233, '2023-05-25 12:06:35.000000', 137, 14533, 13, 0.09058791972962155),
(97434, '2023-05-25 12:39:46.000000', 137, 14657, 13, 0.023315916140422388),
(108167, '2023-05-25 15:41:30.000000', 137, 14964, 13, 0.21757999385617666),
(107741, '2023-05-25 14:40:37.000000', 137, 14964, 13, 0.3449447147508202),
(106312, '2023-05-25 14:40:16.000000', 137, 14964, 13, 0.11890456868959376),
(106134, '2023-05-25 12:56:18.000000', 137, 14964, 13, 0.8004332371337775),
(103696, '2023-05-25 12:54:31.000000', 137, 14964, 13, 0.30147495793613643),
(106311, '2023-05-25 12:44:22.000000', 137, 14964, 13, 0.7412968055185551),
(106133, '2023-05-25 12:44:22.000000', 137, 14964, 13, 0.12940337622720932),
(105711, '2023-05-25 12:43:57.000000', 137, 14964, 13, 0.1044849979830822),
(105710, '2023-05-25 12:34:04.000000', 137, 14964, 13, 0.9113563410974876),
(108787, '2023-05-25 17:59:35.000000', 137, 15377, 13, 0.921829256160489),
(107833, '2023-05-25 14:53:08.000000', 137, 16302, 13, 0.9663117845438407),
(105435, '2023-05-25 12:30:59.000000', 137, 16568, 13, 0.13774896612028797),
(105434, '2023-05-25 12:29:29.000000', 137, 16568, 13, 0.3891495411502035),
(108357, '2023-05-25 16:18:39.000000', 137, 16665, 13, 0.44701901843246716),
(98564, '2023-05-25 17:12:43.000000', 138, 14760, 13, 0.8463114782142114),
(109032, '2023-05-25 19:00:00.000000', 138, 14992, 13, 0.025578609447126865),
(108800, '2023-05-25 18:43:18.000000', 138, 14992, 13, 0.5397724043221928),
(108799, '2023-05-25 18:00:00.000000', 138, 14992, 13, 0.0321658507434357),
(107320, '2023-05-25 14:00:01.000000', 138, 14992, 13, 0.9042941365487067),
(107296, '2023-05-25 14:00:00.000000', 138, 14992, 13, 0.7821178685669885),
(104700, '2023-05-25 12:36:55.000000', 138, 14992, 13, 0.6854496458178119),
(105177, '2023-05-25 12:00:01.000000', 138, 14992, 13, 0.23780719110724746),
(109330, '2023-05-25 23:59:13.000000', 138, 15080, 13, 0.5409015970284159),
(107400, '2023-05-25 16:45:13.000000', 138, 15080, 13, 0.6233594483468217),
(107399, '2023-05-25 14:03:49.000000', 138, 15080, 13, 0.8192327792045404),
(105004, '2023-05-25 13:37:49.000000', 138, 15080, 13, 0.2993620446103442),
(102592, '2023-05-25 13:31:48.000000', 138, 15080, 13, 0.24649704579496046),
(109028, '2023-05-25 19:00:00.000000', 138, 15123, 13, 0.5442767942906279),
(108794, '2023-05-25 18:43:18.000000', 138, 15123, 13, 0.29095714680616425),
(108793, '2023-05-25 18:00:00.000000', 138, 15123, 13, 0.681894893772391),
(107314, '2023-05-25 14:00:01.000000', 138, 15123, 13, 0.9637603904838059),
(107292, '2023-05-25 14:00:00.000000', 138, 15123, 13, 0.05956707862994293),
(104696, '2023-05-25 12:36:55.000000', 138, 15123, 13, 0.27039489547807705),
(105171, '2023-05-25 12:00:01.000000', 138, 15123, 13, 0.1269705046788907),
(106625, '2023-05-25 13:22:19.000000', 138, 15326, 13, 0.7712280764026431),
(106624, '2023-05-25 13:15:49.000000', 138, 15326, 13, 0.585381418741779),
(105699, '2023-05-25 12:44:33.000000', 138, 15326, 13, 0.3710994669938259),
(105698, '2023-05-25 12:33:41.000000', 138, 15326, 13, 0.8992328857980105),
(108514, '2023-05-25 16:47:37.000000', 138, 15620, 13, 0.40346934167556725),
(102691, '2023-05-25 13:33:57.000000', 138, 15620, 13, 0.8046719989908304),
(103655, '2023-05-25 13:34:39.000000', 138, 15740, 13, 0.2541099322817928),
(106987, '2023-05-25 13:37:36.000000', 138, 15766, 13, 0.8407818724583045),
(102180, '2023-05-25 13:37:11.000000', 138, 15766, 13, 0.19149633299917213),
(102717, '2023-05-25 13:38:17.000000', 138, 15868, 13, 0.03196157886032225),
(102719, '2023-05-25 13:38:42.000000', 138, 15921, 13, 0.9986438564169191),
(103659, '2023-05-25 13:35:11.000000', 138, 15926, 13, 0.8549591705597201),
(108796, '2023-05-25 18:43:18.000000', 138, 15932, 13, 0.6213586835883191),
(108795, '2023-05-25 18:00:00.000000', 138, 15932, 13, 0.6730718577847092),
(107326, '2023-05-25 14:00:01.000000', 138, 15932, 13, 0.278131899094646),
(107298, '2023-05-25 14:00:00.000000', 138, 15932, 13, 0.92423751071723),
(104702, '2023-05-25 12:36:55.000000', 138, 15932, 13, 0.22221315122722984),
(105175, '2023-05-25 12:00:01.000000', 138, 15932, 13, 0.28839114292751233),
(102736, '2023-05-25 16:15:29.000000', 138, 16052, 13, 0.431037595792759),
(99163, '2023-05-25 13:47:30.000000', 138, 16419, 13, 0.5291021511946319),
(102738, '2023-05-25 13:45:05.000000', 138, 16420, 13, 0.6506497895856924),
(99109, '2023-05-25 13:37:49.000000', 138, 16420, 13, 0.019501542758906254),
(108798, '2023-05-25 18:43:18.000000', 138, 16590, 13, 0.8990882904615916),
(108797, '2023-05-25 18:00:00.000000', 138, 16590, 13, 0.8888186371755147),
(107328, '2023-05-25 14:00:01.000000', 138, 16590, 13, 0.019486942610562608),
(107300, '2023-05-25 14:00:00.000000', 138, 16590, 13, 0.5614292991802508),
(104698, '2023-05-25 12:36:55.000000', 138, 16590, 13, 0.01866956387405594),
(105173, '2023-05-25 12:00:01.000000', 138, 16590, 13, 0.25661478763909074),
(107224, '2023-05-25 13:51:57.000000', 138, 16633, 13, 0.0010321723593804677),
(99064, '2023-05-25 13:37:49.000000', 138, 16633, 13, 0.8675616866165861),
(109225, '2023-05-25 22:13:01.000000', 138, 16669, 13, 0.1076822852142385),
(109224, '2023-05-25 21:11:56.000000', 138, 16669, 13, 0.24001365186054713);
SELECT compress_chunk(show_chunks('test', older_than => INTERVAL '1 week'), true);
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_13_25_chunk
(1 row)

SELECT t.dttm FROM test t WHERE t.dttm > '2023-05-25T14:23:12' ORDER BY t.dttm;
dttm
--------------------------
Thu May 25 14:24:12 2023
Thu May 25 14:40:16 2023
Thu May 25 14:40:37 2023
Thu May 25 14:53:08 2023
Thu May 25 15:41:30 2023
Thu May 25 16:09:17 2023
Thu May 25 16:15:29 2023
Thu May 25 16:18:39 2023
Thu May 25 16:45:13 2023
Thu May 25 16:47:37 2023
Thu May 25 17:12:43 2023
Thu May 25 17:59:35 2023
Thu May 25 18:00:00 2023
Thu May 25 18:00:00 2023
Thu May 25 18:00:00 2023
Thu May 25 18:00:00 2023
Thu May 25 18:39:07 2023
Thu May 25 18:43:18 2023
Thu May 25 18:43:18 2023
Thu May 25 18:43:18 2023
Thu May 25 18:43:18 2023
Thu May 25 19:00:00 2023
Thu May 25 19:00:00 2023
Thu May 25 19:59:54 2023
Thu May 25 19:59:54 2023
Thu May 25 19:59:54 2023
Thu May 25 21:11:56 2023
Thu May 25 22:13:01 2023
Thu May 25 23:12:13 2023
Thu May 25 23:12:13 2023
Thu May 25 23:59:13 2023
(31 rows)

0 comments on commit 33a3e10

Please sign in to comment.