Skip to content

Commit

Permalink
Merge pull request #8250 from ClickHouse/no_mutation_number_by_default
Browse files Browse the repository at this point in the history 
No mutation number by default

(cherry picked from commit c9887cc)
  • Loading branch information
@alexey-milovidov @KochetovNicolai
alexey-milovidov authored and KochetovNicolai committed Dec 19, 2019
1 parent 1681805 commit c4905ed
Show file tree
Hide file tree
Showing 16 changed files with 186 additions and 124 deletions.
3 changes: 2 additions & 1 deletion dbms/src/Storages/MergeTree/MergeTreeData.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2027,7 +2027,8 @@ void MergeTreeData::renameTempPartAndReplace(
*/
if (increment)
{
part_info.min_block = part_info.max_block = part_info.mutation = increment->get();
part_info.min_block = part_info.max_block = increment->get();
part_info.mutation = 0; /// it's equal to min_block by default
part_name = part->getNewName(part_info);
}
else
Expand Down
24 changes: 12 additions & 12 deletions dbms/tests/integration/test_check_table/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,35 +50,35 @@ def remove_part_from_disk(node, table, part_name):

def test_check_normal_table_corruption(started_cluster):
node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-02-01'), 1, 10), (toDate('2019-02-01'), 2, 12)")
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}) == "201902_1_1_0_1\t1\t\n"
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}) == "201902_1_1_0\t1\t\n"

remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0_1")
remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0")

assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0_1\t1\tChecksums recounted and written to disk."
assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."

assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n"

remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0_1")
remove_checksums_on_disk(node1, "non_replicated_mt", "201902_1_1_0")

assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0_1\t1\tChecksums recounted and written to disk."
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201902", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t1\tChecksums recounted and written to disk."

assert node1.query("SELECT COUNT() FROM non_replicated_mt") == "2\n"

corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0_1")
corrupt_data_part_on_disk(node1, "non_replicated_mt", "201902_1_1_0")

assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0_1\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."

assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0_1\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."
assert node1.query("CHECK TABLE non_replicated_mt", settings={"check_query_single_value_result": 0}).strip() == "201902_1_1_0\t0\tCannot read all data. Bytes read: 2. Bytes expected: 16."

node1.query("INSERT INTO non_replicated_mt VALUES (toDate('2019-01-01'), 1, 10), (toDate('2019-01-01'), 2, 12)")

assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0_2\t1\t\n"
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t1\t\n"

corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0_2")
corrupt_data_part_on_disk(node1, "non_replicated_mt", "201901_2_2_0")

remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0_2")
remove_checksums_on_disk(node1, "non_replicated_mt", "201901_2_2_0")

assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0_2\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n"
assert node1.query("CHECK TABLE non_replicated_mt PARTITION 201901", settings={"check_query_single_value_result": 0}) == "201901_2_2_0\t0\tCheck of part finished with error: \\'Cannot read all data. Bytes read: 2. Bytes expected: 16.\\'\n"


def test_check_replicated_table_simple(started_cluster):
Expand Down
4 changes: 2 additions & 2 deletions dbms/tests/integration/test_config_substitutions/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ def test_allow_databases(start_cluster):
assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'") == "db1\n"
assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table' ") == "test_table\n"
assert node5.query("SELECT name FROM system.columns WHERE database = 'db1' AND table = 'test_table'") == "date\nk1\nv1\n"
assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0_1\n"
assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0_1\n20000101_20000101_1_1_0_1\n20000101_20000101_1_1_0_1\n"
assert node5.query("SELECT name FROM system.parts WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n"
assert node5.query("SELECT name FROM system.parts_columns WHERE database = 'db1' AND table = 'test_table'") == "20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n20000101_20000101_1_1_0\n"

assert node5.query("SELECT name FROM system.databases WHERE name = 'db1'", user="test_allow").strip() == ""
assert node5.query("SELECT name FROM system.tables WHERE database = 'db1' AND name = 'test_table'", user="test_allow").strip() == ""
Expand Down
5 changes: 2 additions & 3 deletions dbms/tests/integration/test_filesystem_layout/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,5 @@ def test_file_path_escaping(started_cluster):
node.query('''INSERT INTO test.`T.a_b,l-e!` VALUES (1);''')
node.query('''ALTER TABLE test.`T.a_b,l-e!` FREEZE;''')

node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0_1/%7EId.bin"])
node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0_1/%7EId.bin"])

node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])
node.exec_in_container(["bash", "-c", "test -f /var/lib/clickhouse/shadow/1/data/test/T%2Ea_b%2Cl%2De%21/1_1_1_0/%7EId.bin"])
10 changes: 5 additions & 5 deletions dbms/tests/integration/test_insert_into_distributed/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,11 @@ def test_inserts_batching(started_cluster):
# 4. Full batch of inserts after ALTER (that have different block structure).
# 5. What was left to insert with the column structure before ALTER.
expected = '''\
20000101_20000101_1_1_0_1\t[1]
20000101_20000101_2_2_0_2\t[2,3,4]
20000101_20000101_3_3_0_3\t[5,6,7]
20000101_20000101_4_4_0_4\t[10,11,12]
20000101_20000101_5_5_0_5\t[8,9]
20000101_20000101_1_1_0\t[1]
20000101_20000101_2_2_0\t[2,3,4]
20000101_20000101_3_3_0\t[5,6,7]
20000101_20000101_4_4_0\t[10,11,12]
20000101_20000101_5_5_0\t[8,9]
'''
assert TSV(result) == TSV(expected)

Expand Down
85 changes: 42 additions & 43 deletions dbms/tests/integration/test_partition/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,30 +65,30 @@ def partition_complex_assert_checksums():
" | sort" \
" | uniq"

checksums = "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700102_2_2_0_2/k.bin\n" \
"082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700201_1_1_0_1/v1.bin\n" \
"13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition/19700102_2_2_0_2/minmax_p.idx\n" \
"25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition/19700102_2_2_0_2/partition.dat\n" \
"3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition/19700201_1_1_0_1/partition.dat\n" \
"37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition/19700102_2_2_0_2/checksums.txt\n" \
"38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition/19700102_2_2_0_2/v1.bin\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0_2/k.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0_2/p.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0_2/v1.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0_1/k.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0_1/p.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0_1/v1.mrk\n" \
"55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition/19700201_1_1_0_1/primary.idx\n" \
"5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition/19700201_1_1_0_1/checksums.txt\n" \
"77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700102_2_2_0_2/columns.txt\n" \
"77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700201_1_1_0_1/columns.txt\n" \
"88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition/19700201_1_1_0_1/p.bin\n" \
"9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition/19700102_2_2_0_2/primary.idx\n" \
"c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700102_2_2_0_2/count.txt\n" \
"c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700201_1_1_0_1/count.txt\n" \
"cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition/19700102_2_2_0_2/p.bin\n" \
"e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition/19700201_1_1_0_1/k.bin\n" \
"f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition/19700201_1_1_0_1/minmax_p.idx\n"
checksums = "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700102_2_2_0/k.bin\n" \
"082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition/19700201_1_1_0/v1.bin\n" \
"13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition/19700102_2_2_0/minmax_p.idx\n" \
"25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition/19700102_2_2_0/partition.dat\n" \
"3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition/19700201_1_1_0/partition.dat\n" \
"37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition/19700102_2_2_0/checksums.txt\n" \
"38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition/19700102_2_2_0/v1.bin\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/k.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/p.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700102_2_2_0/v1.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/k.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/p.mrk\n" \
"4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition/19700201_1_1_0/v1.mrk\n" \
"55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition/19700201_1_1_0/primary.idx\n" \
"5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition/19700201_1_1_0/checksums.txt\n" \
"77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700102_2_2_0/columns.txt\n" \
"77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition/19700201_1_1_0/columns.txt\n" \
"88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition/19700201_1_1_0/p.bin\n" \
"9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition/19700102_2_2_0/primary.idx\n" \
"c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700102_2_2_0/count.txt\n" \
"c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition/19700201_1_1_0/count.txt\n" \
"cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition/19700102_2_2_0/p.bin\n" \
"e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition/19700201_1_1_0/k.bin\n" \
"f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition/19700201_1_1_0/minmax_p.idx\n"

assert TSV(exec_bash(cmd).replace(' ', '\t')) == TSV(checksums)

Expand Down Expand Up @@ -149,7 +149,7 @@ def test_cannot_attach_active_part(cannot_attach_active_part_table):
assert 0 <= error.find('Invalid part name')

res = q("SElECT name FROM system.parts WHERE table='attach_active' AND database='test' ORDER BY name")
assert TSV(res) == TSV('0_1_1_0_1\n1_2_2_0_2\n2_3_3_0_3\n3_4_4_0_4')
assert TSV(res) == TSV('0_1_1_0\n1_2_2_0\n2_3_3_0\n3_4_4_0')
assert TSV(q("SElECT count(), sum(n) FROM test.attach_active")) == TSV('16\t120')


Expand All @@ -171,30 +171,30 @@ def test_attach_check_all_parts(attach_check_all_parts_table):
q("ALTER TABLE test.attach_partition DETACH PARTITION 0")

path_to_detached = path_to_data + 'data/test/attach_partition/detached/'
exec_bash('mkdir {}'.format(path_to_detached + '0_5_5_0_5'))
exec_bash('cp -pr {} {}'.format(path_to_detached + '0_1_1_0_1', path_to_detached + 'attaching_0_6_6_0_6'))
exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0_3', path_to_detached + 'deleting_0_7_7_0_7'))
exec_bash('mkdir {}'.format(path_to_detached + '0_5_5_0'))
exec_bash('cp -pr {} {}'.format(path_to_detached + '0_1_1_0', path_to_detached + 'attaching_0_6_6_0'))
exec_bash('cp -pr {} {}'.format(path_to_detached + '0_3_3_0', path_to_detached + 'deleting_0_7_7_0'))

error = instance.client.query_and_get_error("ALTER TABLE test.attach_partition ATTACH PARTITION 0")
assert 0 <= error.find('No columns in part 0_5_5_0')

parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name")
assert TSV(parts) == TSV('1_2_2_0_2\n1_4_4_0_4')
assert TSV(parts) == TSV('1_2_2_0\n1_4_4_0')
detached = q("SELECT name FROM system.detached_parts "
"WHERE table='attach_partition' AND database='test' ORDER BY name")
assert TSV(detached) == TSV('0_1_1_0_1\n0_3_3_0_3\n0_5_5_0_5\nattaching_0_6_6_0_6\ndeleting_0_7_7_0_7')
assert TSV(detached) == TSV('0_1_1_0\n0_3_3_0\n0_5_5_0\nattaching_0_6_6_0\ndeleting_0_7_7_0')

exec_bash('rm -r {}'.format(path_to_detached + '0_5_5_0_5'))
exec_bash('rm -r {}'.format(path_to_detached + '0_5_5_0'))

q("ALTER TABLE test.attach_partition ATTACH PARTITION 0")
parts = q("SElECT name FROM system.parts WHERE table='attach_partition' AND database='test' ORDER BY name")
expected = '0_5_5_0_5\n0_6_6_0_6\n1_2_2_0_2\n1_4_4_0_4'
expected = '0_5_5_0\n0_6_6_0\n1_2_2_0\n1_4_4_0'
assert TSV(parts) == TSV(expected)
assert TSV(q("SElECT count(), sum(n) FROM test.attach_partition")) == TSV('16\t120')

detached = q("SELECT name FROM system.detached_parts "
"WHERE table='attach_partition' AND database='test' ORDER BY name")
assert TSV(detached) == TSV('attaching_0_6_6_0_6\ndeleting_0_7_7_0_7')
assert TSV(detached) == TSV('attaching_0_6_6_0\ndeleting_0_7_7_0')


@pytest.fixture
Expand All @@ -217,28 +217,27 @@ def test_drop_detached_parts(drop_detached_parts_table):
q("ALTER TABLE test.drop_detached DETACH PARTITION 1")

path_to_detached = path_to_data + 'data/test/drop_detached/detached/'
exec_bash('mkdir {}'.format(path_to_detached + 'attaching_0_6_6_0_6'))
exec_bash('mkdir {}'.format(path_to_detached + 'deleting_0_7_7_0_7'))
exec_bash('mkdir {}'.format(path_to_detached + 'attaching_0_6_6_0'))
exec_bash('mkdir {}'.format(path_to_detached + 'deleting_0_7_7_0'))
exec_bash('mkdir {}'.format(path_to_detached + 'any_other_name'))
exec_bash('mkdir {}'.format(path_to_detached + 'prefix_1_2_2_0_2'))
exec_bash('mkdir {}'.format(path_to_detached + 'prefix_1_2_2_0_0'))

error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0_2'", settings=s)
error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART '../1_2_2_0'", settings=s)
assert 0 <= error.find('Invalid part name')

q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0_1'", settings=s)
q("ALTER TABLE test.drop_detached DROP DETACHED PART '0_1_1_0'", settings=s)

error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0_6'", settings=s)
error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'attaching_0_6_6_0'", settings=s)
assert 0 <= error.find('Cannot drop part')

error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0_7'", settings=s)
error = instance.client.query_and_get_error("ALTER TABLE test.drop_detached DROP DETACHED PART 'deleting_0_7_7_0'", settings=s)
assert 0 <= error.find('Cannot drop part')

q("ALTER TABLE test.drop_detached DROP DETACHED PART 'any_other_name'", settings=s)

detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name")
assert TSV(detached) == TSV('0_3_3_0_3\n1_2_2_0_2\n1_4_4_0_4\nattaching_0_6_6_0_6\ndeleting_0_7_7_0_7\nprefix_1_2_2_0_2')
assert TSV(detached) == TSV('0_3_3_0\n1_2_2_0\n1_4_4_0\nattaching_0_6_6_0\ndeleting_0_7_7_0\nprefix_1_2_2_0_0')

q("ALTER TABLE test.drop_detached DROP DETACHED PARTITION 1", settings=s)
detached = q("SElECT name FROM system.detached_parts WHERE table='drop_detached' AND database='test' ORDER BY name")
assert TSV(detached) == TSV('0_3_3_0_3\nattaching_0_6_6_0_6\ndeleting_0_7_7_0_7')

assert TSV(detached) == TSV('0_3_3_0\nattaching_0_6_6_0\ndeleting_0_7_7_0')
Loading

0 comments on commit c4905ed

Please sign in to comment.