-
Notifications
You must be signed in to change notification settings - Fork 6.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #42122 from ClickHouse/backport/22.9/41741
Backport #41741 to 22.9: Fix intersecting parts
- Loading branch information
Showing
14 changed files
with
353 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
11 changes: 11 additions & 0 deletions
11
tests/queries/0_stateless/02369_lost_part_intersecting_merges.reference
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
1 1 all_0_1_1 | ||
1 2 all_0_1_1 | ||
2 1 all_0_0_0 | ||
2 2 all_1_1_0 | ||
0 | ||
3 1 all_0_2_2 | ||
3 2 all_0_2_2 | ||
3 3 all_0_2_2 | ||
4 1 all_0_2_2 | ||
4 2 all_0_2_2 | ||
4 3 all_0_2_2 |
46 changes: 46 additions & 0 deletions
46
tests/queries/0_stateless/02369_lost_part_intersecting_merges.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
#!/usr/bin/env bash | ||
# Tags: zookeeper | ||
|
||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) | ||
# shellcheck source=../shell_config.sh | ||
. "$CURDIR"/../shell_config.sh | ||
|
||
$CLICKHOUSE_CLIENT -q "drop table if exists rmt1;" | ||
$CLICKHOUSE_CLIENT -q "drop table if exists rmt2;" | ||
|
||
$CLICKHOUSE_CLIENT -q "create table rmt1 (n int) engine=ReplicatedMergeTree('/test/02369/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{database}', '1') order by n;" | ||
$CLICKHOUSE_CLIENT -q "create table rmt2 (n int) engine=ReplicatedMergeTree('/test/02369/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{database}', '2') order by n;" | ||
|
||
$CLICKHOUSE_CLIENT -q "insert into rmt1 values (1);" | ||
$CLICKHOUSE_CLIENT -q "insert into rmt1 values (2);" | ||
|
||
$CLICKHOUSE_CLIENT -q "system sync replica rmt1;" | ||
$CLICKHOUSE_CLIENT -q "system sync replica rmt2;" | ||
$CLICKHOUSE_CLIENT -q "system stop merges rmt2;" | ||
$CLICKHOUSE_CLIENT -q "optimize table rmt1 final;" | ||
|
||
$CLICKHOUSE_CLIENT -q "select 1, *, _part from rmt1 order by n;" | ||
$CLICKHOUSE_CLIENT -q "select 2, *, _part from rmt2 order by n;" | ||
|
||
path=$($CLICKHOUSE_CLIENT -q "select path from system.parts where database='$CLICKHOUSE_DATABASE' and table='rmt1' and name='all_0_1_1'") | ||
# ensure that path is absolute before removing | ||
$CLICKHOUSE_CLIENT -q "select throwIf(substring('$path', 1, 1) != '/', 'Path is relative: $path')" || exit | ||
rm -rf $path | ||
|
||
$CLICKHOUSE_CLIENT -q "select * from rmt1;" 2>/dev/null | ||
|
||
$CLICKHOUSE_CLIENT -q "detach table rmt1;" | ||
$CLICKHOUSE_CLIENT -q "attach table rmt1;" | ||
|
||
$CLICKHOUSE_CLIENT -q "insert into rmt1 values (3);" | ||
$CLICKHOUSE_CLIENT -q "system start merges rmt2;" | ||
$CLICKHOUSE_CLIENT -q "system sync replica rmt1;" | ||
$CLICKHOUSE_CLIENT -q "optimize table rmt1 final;" | ||
|
||
$CLICKHOUSE_CLIENT -q "system sync replica rmt1;" | ||
$CLICKHOUSE_CLIENT -q "system sync replica rmt2;" | ||
$CLICKHOUSE_CLIENT -q "select 3, *, _part from rmt1 order by n;" | ||
$CLICKHOUSE_CLIENT -q "select 4, *, _part from rmt2 order by n;" | ||
|
||
$CLICKHOUSE_CLIENT -q "drop table rmt1;" | ||
$CLICKHOUSE_CLIENT -q "drop table rmt2;" |
2 changes: 2 additions & 0 deletions
2
tests/queries/0_stateless/02396_system_parts_race_condition_rm.reference
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Replication did not hang: synced all replicas of alter_table | ||
Consistency: 1 |
101 changes: 101 additions & 0 deletions
101
tests/queries/0_stateless/02396_system_parts_race_condition_rm.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
#!/usr/bin/env bash | ||
# Tags: race, zookeeper, no-parallel, disabled | ||
|
||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) | ||
# shellcheck source=../shell_config.sh | ||
. "$CURDIR"/../shell_config.sh | ||
# shellcheck source=./replication.lib | ||
. "$CURDIR"/replication.lib | ||
|
||
set -e | ||
|
||
# NOTE this test is copy of 00992_system_parts_race_condition_zookeeper_long, but with extra thread7 | ||
|
||
$CLICKHOUSE_CLIENT -n -q " | ||
DROP TABLE IF EXISTS alter_table0; | ||
DROP TABLE IF EXISTS alter_table1; | ||
CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; | ||
CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 | ||
" | ||
|
||
function thread1() | ||
{ | ||
# NOTE: database = $CLICKHOUSE_DATABASE is unwanted | ||
while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; done | ||
} | ||
|
||
function thread2() | ||
{ | ||
while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; ALTER TABLE alter_table0 DROP COLUMN h;"; done | ||
} | ||
|
||
function thread3() | ||
{ | ||
while true; do $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table0 SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; done | ||
} | ||
|
||
function thread4() | ||
{ | ||
while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL"; done | ||
} | ||
|
||
function thread5() | ||
{ | ||
while true; do $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table0 DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288"; done | ||
} | ||
|
||
function thread7() | ||
{ | ||
while true; do | ||
path=$($CLICKHOUSE_CLIENT -q "SELECT path FROM system.parts WHERE database='$CLICKHOUSE_DATABASE' AND table LIKE 'alter_table%' ORDER BY rand() LIMIT 1") | ||
if [ -z "$path" ]; then continue; fi | ||
# ensure that path is absolute before removing | ||
$CLICKHOUSE_CLIENT -q "select throwIf(substring('$path', 1, 1) != '/', 'Path is relative: $path') format Null" || exit | ||
rm -rf $path 2> /dev/null | ||
sleep 0.$RANDOM; | ||
done | ||
} | ||
|
||
# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout | ||
export -f thread1; | ||
export -f thread2; | ||
export -f thread3; | ||
export -f thread4; | ||
export -f thread5; | ||
export -f thread7; | ||
|
||
TIMEOUT=10 | ||
|
||
timeout $TIMEOUT bash -c thread1 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread2 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread3 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread4 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread5 2> /dev/null & | ||
|
||
timeout $TIMEOUT bash -c thread1 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread2 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread3 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread4 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread5 2> /dev/null & | ||
|
||
timeout $TIMEOUT bash -c thread1 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread2 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread3 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread4 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread5 2> /dev/null & | ||
|
||
timeout $TIMEOUT bash -c thread1 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread2 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread3 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread4 2> /dev/null & | ||
timeout $TIMEOUT bash -c thread5 2> /dev/null & | ||
|
||
timeout $TIMEOUT bash -c thread7 & | ||
|
||
wait | ||
check_replication_consistency "alter_table" "count(), sum(a), sum(b), round(sum(c))" | ||
|
||
$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & | ||
$CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & | ||
wait |
2 changes: 2 additions & 0 deletions
2
tests/queries/0_stateless/02397_system_parts_race_condition_drop_rm.reference
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Replication did not hang: synced all replicas of alter_table_ | ||
Consistency: 1 |
Oops, something went wrong.