Check simple dictionary key is native unsigned integer

docs/en/operations/settings/settings.md

@@ -4071,6 +4071,36 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca
 Possible values: `true`, `false`
 
 Default value: `false`
+
+## check_dictionary_primary_key {#check_dictionary_primary_key}
+
+Enables the check at dictionay creation, dictionaries without word complex-key* in a layout have a key with UInt64 type. The primary key data type must be one of unsigned [integer types](../../sql-reference/data-types/int-uint.md): `UInt8`, `UInt16`, `UInt32`, `UInt64`.
+Possible values:
+
+-   true  — The check is enabled.
+-   false — The check is disabled at dictionay creation.
+
+Default value: `true`.
+
+If you already have dictionay with incorrect primar key and do not want the server to raise an exception during startup, set `check_dictionary_primary_key` to `false`.
+
+Or you can create dictionay with settings `check_dictionary_primary_key` to `false`.
+
+**Example**
+
+```sql
+CREATE DICTIONARY test
+(
+    `id` Int128,
+    `name` String
+)
+PRIMARY KEY id
+SOURCE(CLICKHOUSE(TABLE 'test_local'))
+LIFETIME(MIN 0 MAX 300)
+LAYOUT(HASHED())
+SETTINGS(check_dictionary_primary_key = 0);
+```
+
 ## function_json_value_return_type_allow_nullable
 
 Control whether allow to return `NULL` when value is not exist for JSON_VALUE function.

src/Core/Settings.h

@@ -951,6 +951,7 @@ class IColumn;
     M(Bool, regexp_dict_allow_hyperscan, true, "Allow regexp_tree dictionary using Hyperscan library.", 0) \
     \
     M(Bool, dictionary_use_async_executor, false, "Execute a pipeline for reading from a dictionary with several threads. It's supported only by DIRECT dictionary with CLICKHOUSE source.", 0) \
+    M(Bool, check_dictionary_primary_key, true, "Check primary key type for simple dictionary is native unsigned integer", 0) \
 
 // End of FORMAT_FACTORY_SETTINGS
 // Please add settings non-related to formats into the COMMON_SETTINGS above.

src/Dictionaries/getDictionaryConfigurationFromAST.cpp

@@ -19,6 +19,7 @@
 #include <Functions/FunctionFactory.h>
 #include <Common/isLocalAddress.h>
 #include <Interpreters/Context.h>
+#include <DataTypes/DataTypeFactory.h>
 
 
 namespace DB
@@ -341,7 +342,9 @@ void buildPrimaryKeyConfiguration(
     AutoPtr<Element> root,
     bool complex,
     const Names & key_names,
-    const ASTExpressionList * dictionary_attributes)
+    const ASTExpressionList * dictionary_attributes,
+    const ASTDictionarySettings * dict_settings,
+    ContextPtr context)
 {
     const auto & children = dictionary_attributes->children;
 
@@ -376,6 +379,26 @@ void buildPrimaryKeyConfiguration(
 
         const ASTDictionaryAttributeDeclaration * dict_attr = (*it)->as<const ASTDictionaryAttributeDeclaration>();
 
+        auto key_type = DataTypeFactory::instance().tryGet(dict_attr->type);
+
+        auto check_dictionary_primary_key = context->getSettingsRef().check_dictionary_primary_key;
+
+        if (dict_settings)
+        {
+            if (const auto * check_dictionary_primary_key_change = dict_settings->changes.tryGet("check_dictionary_primary_key"))
+            {
+                check_dictionary_primary_key = check_dictionary_primary_key_change->get<bool>();
+            }
+        }
+
+        if (check_dictionary_primary_key && !WhichDataType(key_type).isNativeUInt())
+        {
+            throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION,
+                "Invalid Primary key type for simple dictionary: {}. Must be native unsigned integer type. "
+                "To avoid checking it, please set check_dictionary_primary_key=false",
+                dict_attr->name);
+        }
+
         AutoPtr<Text> name(doc->createTextNode(dict_attr->name));
         name_element->appendChild(name);
 
@@ -614,7 +637,7 @@ getDictionaryConfigurationFromAST(const ASTCreateQuery & query, ContextPtr conte
 
     checkPrimaryKey(all_attr_names_and_types, pk_attrs);
 
-    buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list);
+    buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list, query.dictionary->dict_settings, context);
 
     buildLayoutConfiguration(xml_document, current_dictionary, query.dictionary->dict_settings, dictionary_layout);
     buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source, query.dictionary->dict_settings, context);

tests/integration/test_backup_restore_new/test.py

@@ -689,7 +689,7 @@ def test_dependencies():
         "CREATE DICTIONARY test.dict1(x UInt32, w String) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB 'test' TABLE 'view')) LAYOUT(FLAT()) LIFETIME(0)"
     )
     instance.query(
-        "CREATE DICTIONARY test.dict2(x UInt32, w String) PRIMARY KEY w SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB 'test' TABLE 'dict1')) LAYOUT(FLAT()) LIFETIME(0)"
+        "CREATE DICTIONARY test.dict2(x UInt32, w String) PRIMARY KEY w SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB 'test' TABLE 'dict1')) LAYOUT(FLAT()) LIFETIME(0) SETTINGS(check_dictionary_primary_key = 0)"
     )
     instance.query(
         "CREATE TABLE test.table2(k String, v Int32 DEFAULT dictGet('test.dict2', 'x', k) - 1) ENGINE=MergeTree ORDER BY tuple()"
@@ -1421,7 +1421,8 @@ def test_tables_dependency():
     instance.query(f"CREATE MATERIALIZED VIEW {t3} TO {t2} AS SELECT x, y FROM {t1}")
 
     instance.query(
-        f"CREATE DICTIONARY {t4} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) LAYOUT(FLAT()) LIFETIME(4)"
+        f"CREATE DICTIONARY {t4} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) "
+        f"LAYOUT(FLAT()) LIFETIME(4) SETTINGS(check_dictionary_primary_key = 0)"
     )
 
     instance.query(f"CREATE TABLE {t5} AS dictionary({t4})")
@@ -1433,7 +1434,8 @@ def test_tables_dependency():
     instance.query(f"CREATE VIEW {t7} AS SELECT sum(x) FROM (SELECT x FROM {t6})")
 
     instance.query(
-        f"CREATE DICTIONARY {t8} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) LAYOUT(FLAT()) LIFETIME(9)"
+        f"CREATE DICTIONARY {t8} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) "
+        f"LAYOUT(FLAT()) LIFETIME(9) SETTINGS(check_dictionary_primary_key = 0)"
     )
 
     instance.query(f"CREATE TABLE {t9}(a Int64) ENGINE=Log")

tests/integration/test_backup_restore_on_cluster/test.py

@@ -884,7 +884,7 @@ def test_tables_dependency():
 
     node1.query(
         "CREATE DICTIONARY mydb.dict ON CLUSTER 'cluster' (x Int64, y String) PRIMARY KEY x "
-        "SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB 'mydb' TABLE 'src')) LAYOUT(FLAT()) LIFETIME(0)"
+        "SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB 'mydb' TABLE 'src')) LAYOUT(FLAT()) LIFETIME(0) SETTINGS(check_dictionary_primary_key = 0)"
     )
 
     node1.query(

tests/integration/test_backward_compatibility/test_convert_ordinary.py

@@ -116,12 +116,16 @@ def create_some_tables(db):
     node.query(
         "CREATE DICTIONARY {}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n "
         "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB '{}')) "
-        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())".format(db, db)
+        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0)".format(
+            db, db
+        )
     )
     node.query(
         "CREATE DICTIONARY {}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n "
         "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB '{}')) "
-        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())".format(db, db)
+        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0)".format(
+            db, db
+        )
     )
     node.query(
         "CREATE TABLE {}.merge (n int) ENGINE=Merge('{}', '(mt)|(mv)')".format(db, db)

tests/integration/test_dictionaries_access/test.py

@@ -33,7 +33,7 @@ def clear_after_test():
     CREATE DICTIONARY test_dict(x Int32, y Int32) PRIMARY KEY x
     LAYOUT(FLAT())
     SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_table' DB 'default'))
-    LIFETIME(0)
+    LIFETIME(0) SETTINGS(check_dictionary_primary_key = 0)
     """
 
 drop_query = "DROP DICTIONARY test_dict"

tests/integration/test_dictionaries_ddl/test.py

@@ -156,6 +156,7 @@ def test_create_and_select_mysql(started_cluster, clickhouse, name, layout):
     ))
     {}
     LIFETIME(MIN 1 MAX 3)
+    SETTINGS(check_dictionary_primary_key = 0)
     """.format(
             name, name, layout
         )
@@ -398,6 +399,7 @@ def test_dictionary_with_where(started_cluster):
     ))
     LAYOUT(FLAT())
     LIFETIME(MIN 1 MAX 3)
+    SETTINGS(check_dictionary_primary_key = 0)
     """
     )
 

tests/integration/test_dictionaries_dependency_xml/test.py

@@ -114,7 +114,7 @@ def test_dependent_tables(started_cluster):
     query(
         "create dictionary a.d (n int default 0, m int default 42) primary key n "
         "source(clickhouse(host 'localhost' port tcpPort() user 'default' table 'src' password '' db 'lazy'))"
-        "lifetime(min 1 max 10) layout(flat())"
+        "lifetime(min 1 max 10) layout(flat()) SETTINGS(check_dictionary_primary_key = 0)"
     )
     query("create table system.join (n int, m int) engine=Join(any, left, n)")
     query("insert into system.join values (1, 1)")
@@ -126,7 +126,7 @@ def test_dependent_tables(started_cluster):
     query(
         "create dictionary test.d (n int default 0, m int default 42) primary key n "
         "source(clickhouse(host 'localhost' port tcpPort() user 'default' table 'src' password '' db 'default'))"
-        "lifetime(min 1 max 10) layout(flat())"
+        "lifetime(min 1 max 10) layout(flat()) SETTINGS(check_dictionary_primary_key = 0)"
     )
     query(
         "create table join (n int, m default dictGet('a.d', 'm', toUInt64(3)),"

tests/integration/test_mask_sensitive_info/configs/disable_check_dictionary_primary_key.xml

@@ -0,0 +1,7 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <check_dictionary_primary_key>0</check_dictionary_primary_key>
+        </default>
+    </profiles>
+</clickhouse>

tests/integration/test_mask_sensitive_info/test.py

@@ -9,6 +9,9 @@
     main_configs=[
         "configs/named_collections.xml",
     ],
+    user_configs=[
+        "configs/disable_check_dictionary_primary_key.xml",
+    ],
     with_zookeeper=True,
 )
 

tests/integration/test_replicated_database/test.py

@@ -698,12 +698,12 @@ def create_some_tables(db):
     main_node.query(
         f"CREATE DICTIONARY {db}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n "
         "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) "
-        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())"
+        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0)"
     )
     dummy_node.query(
         f"CREATE DICTIONARY {db}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n "
         "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) "
-        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())"
+        "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0)"
     )
 
 
@@ -766,7 +766,7 @@ def test_recover_staled_replica(started_cluster):
         main_node.query_with_retry(
             "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n "
             "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) "
-            "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());",
+            "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0);",
             settings=settings,
         )
 

tests/queries/0_stateless/00962_live_view_periodic_refresh_dictionary.py

@@ -38,7 +38,7 @@
     client1.expect(prompt)
     client1.send(
         "CREATE DICTIONARY test.dict(a Int32, b Int32) PRIMARY KEY a LAYOUT(FLAT()) "
-        + "SOURCE(CLICKHOUSE(db 'test' table 'mt')) LIFETIME(1)"
+        + "SOURCE(CLICKHOUSE(db 'test' table 'mt')) LIFETIME(1) SETTINGS(check_dictionary_primary_key = 0)"
     )
     client1.expect(prompt)
     client1.send("CREATE LIVE VIEW test.lv WITH REFRESH 1 AS SELECT * FROM test.dict")

tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh

@@ -23,7 +23,8 @@ CREATE DICTIONARY dictdb_01042.dict
 PRIMARY KEY x
 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table' DB 'dictdb_01042' UPDATE_FIELD 'insert_time'))
 LAYOUT(FLAT())
-LIFETIME(1);
+LIFETIME(1)
+SETTINGS(check_dictionary_primary_key = 0);
 EOF
 
 $CLICKHOUSE_CLIENT --query "SELECT '12 -> ', dictGetInt64('dictdb_01042.dict', 'y', toUInt64(12))"

tests/queries/0_stateless/01043_dictionary_attribute_properties_values.sql

@@ -17,7 +17,8 @@ CREATE DICTIONARY dictdb_01043.dict
 PRIMARY KEY key
 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dicttbl' DB 'dictdb_01043'))
 LAYOUT(FLAT())
-LIFETIME(1);
+LIFETIME(1)
+SETTINGS(check_dictionary_primary_key = 0);
 
 
 SELECT dictGetString('dictdb_01043.dict', 'value_default', toUInt64(12));

tests/queries/0_stateless/01160_table_dependencies.sh

@@ -16,14 +16,14 @@ $CLICKHOUSE_CLIENT -q "create table dict_src (n int, m int, s String) engine=Mer
 $CLICKHOUSE_CLIENT -q "create dictionary dict1 (n int default 0, m int default 1, s String default 'qqq')
 PRIMARY KEY n
 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dict_src' PASSWORD '' DB '$CLICKHOUSE_DATABASE'))
-LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());"
+LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0);"
 
 $CLICKHOUSE_CLIENT -q "create table join(n int, m int default dictGet('$CLICKHOUSE_DATABASE.dict1', 'm', 42::UInt64)) engine=Join(any, left, n);"
 
 $CLICKHOUSE_CLIENT -q "create dictionary dict2 (n int default 0, m int DEFAULT 2)
 PRIMARY KEY n
 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'join' PASSWORD '' DB '$CLICKHOUSE_DATABASE'))
-LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());"
+LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0);"
 
 $CLICKHOUSE_CLIENT -q "create table s (x default joinGet($CLICKHOUSE_DATABASE.join, 'm', 42::int)) engine=Set"
 
@@ -102,7 +102,7 @@ $CLICKHOUSE_CLIENT -q "create table ${CLICKHOUSE_DATABASE}_1.xdict_src (n int, m
 $CLICKHOUSE_CLIENT -q "create dictionary ${CLICKHOUSE_DATABASE}_1.ydict1 (n int default 0, m int default 1, s String default 'qqq')
 PRIMARY KEY n
 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'xdict_src' PASSWORD '' DB '${CLICKHOUSE_DATABASE}_1'))
-LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());"
+LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(check_dictionary_primary_key = 0);"
 
 $CLICKHOUSE_CLIENT -q "create table ${CLICKHOUSE_DATABASE}_1.zjoin(n int, m int default dictGet('${CLICKHOUSE_DATABASE}_1.ydict1', 'm', 42::UInt64)) engine=Join(any, left, n);"
 $CLICKHOUSE_CLIENT -q "drop database ${CLICKHOUSE_DATABASE}_1"

tests/queries/0_stateless/02364_dictionary_datetime_64_attribute_crash.sql

@@ -9,7 +9,8 @@ CREATE DICTIONARY datDictionary
 PRIMARY KEY blockNum
 SOURCE(CLICKHOUSE(TABLE 'dat'))
 LIFETIME(MIN 0 MAX 1000)
-LAYOUT(FLAT());
+LAYOUT(FLAT())
+SETTINGS(check_dictionary_primary_key = 0);
 
 select (select eventTimestamp from datDictionary);
 select count(*) from dat where eventTimestamp >= (select eventTimestamp from datDictionary);

tests/queries/0_stateless/02390_prometheus_ClickHouseStatusInfo_DictionaryStatus.sh

@@ -13,7 +13,7 @@ function get_dictionary_status()
     }
 }
 
-$CLICKHOUSE_CLIENT -q "CREATE DICTIONARY dict (key Int, value String) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE data)) LAYOUT(HASHED()) LIFETIME(0)"
+$CLICKHOUSE_CLIENT -q "CREATE DICTIONARY dict (key Int, value String) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE data)) LAYOUT(HASHED()) LIFETIME(0) SETTINGS(check_dictionary_primary_key = 0)"
 uuid="$($CLICKHOUSE_CLIENT -q "SELECT uuid FROM system.dictionaries WHERE database = '$CLICKHOUSE_DATABASE' AND name = 'dict'")"
 
 echo 'status before reload'

tests/queries/0_stateless/02391_hashed_dictionary_shards.sql

@@ -87,7 +87,8 @@ CREATE DICTIONARY test_dictionary_10_shards_string
 ) PRIMARY KEY key
 SOURCE(CLICKHOUSE(TABLE test_table_string))
 LAYOUT(SPARSE_HASHED(SHARDS 10))
-LIFETIME(0);
+LIFETIME(0)
+SETTINGS(check_dictionary_primary_key = 0);
 
 SYSTEM RELOAD DICTIONARY test_dictionary_10_shards_string; -- { serverError CANNOT_PARSE_TEXT }
 

tests/queries/0_stateless/02525_range_hashed_dictionary_update_field.sql

@@ -18,7 +18,8 @@ CREATE DICTIONARY test_dictionary
 LAYOUT(RANGE_HASHED())
 RANGE(MIN start MAX end)
 SOURCE(CLICKHOUSE(TABLE 'test_table' UPDATE_FIELD 'insert_time' UPDATE_LAG 10))
-LIFETIME(MIN 1 MAX 2);
+LIFETIME(MIN 1 MAX 2)
+SETTINGS(check_dictionary_primary_key = 0);
 
 INSERT INTO test_table VALUES (1, 0, 100, '2022-12-26 11:38:34'), (1, 101, 200, '2022-12-26 11:38:34'), (2, 0, 999, '2022-12-26 11:38:34'), (2, 1000, 10000, '2022-12-26 11:38:34');
 

tests/queries/0_stateless/02707_check_sample_dict_key_is_correct.sql

@@ -0,0 +1,41 @@
+DROP DICTIONARY IF EXISTS test_sample_key_dict1;
+DROP DICTIONARY IF EXISTS test_sample_key_dict2;
+DROP table IF EXISTS test_sample_key_local;
+
+-- create local table
+CREATE TABLE test_sample_key_local
+(
+    `id` Int128,
+    `name` String
+)
+ENGINE = Memory;
+
+
+-- create DICTIONARY with default settings check_dictionary_primary_key = 1
+CREATE DICTIONARY test_sample_key_dict1
+(
+    `id` Int128,
+    `name` String
+)
+PRIMARY KEY id
+SOURCE(CLICKHOUSE(TABLE 'test_sample_key_local'))
+LIFETIME(MIN 0 MAX 300)
+LAYOUT(HASHED()); -- { serverError 489 }
+
+
+-- create DICTIONARY with settings check_dictionary_primary_key = 0
+CREATE DICTIONARY test_sample_key_dict2
+(
+    `id` Int128,
+    `name` String
+)
+PRIMARY KEY id
+SOURCE(CLICKHOUSE(TABLE 'test_sample_key_local'))
+LIFETIME(MIN 0 MAX 300)
+LAYOUT(HASHED())
+SETTINGS(check_dictionary_primary_key = 0);
+
+
+DROP DICTIONARY IF EXISTS test_sample_key_dict1;
+DROP DICTIONARY IF EXISTS test_sample_key_dict2;
+DROP table IF EXISTS test_sample_key_local;