Skip to content

Commit 5b3ad94

Browse files
dr-mdr-m
committed
MDEV-27208: Extend CRC32() and implement CRC32C()
We used to define a native unary function CRC32() that computes the CRC-32 of a string using the ISO 3309 polynomial that is being used by zlib and many others. Often, a CRC is computed in pieces. To faciliate this, we introduce a 2-ary variant of the function that inputs a previous CRC as the first argument: CRC32('MariaDB')=CRC32(CRC32('Maria'),'DB'). InnoDB and MyRocks use a different polynomial, which was implemented in SSE4.2 instructions that were introduced in the Intel Nehalem microarchitecture. This is commonly called CRC-32C (Castagnoli). We introduce a native function that uses the Castagnoli polynomial: CRC32C('MariaDB')=CRC32C(CRC32C('Maria'),'DB'). This allows SELECT...INTO DUMPFILE to be used for the creation of files with valid checksums, such as a logically empty InnoDB redo log file ib_logfile0 corresponding to a particular log sequence number.
1 parent b07920b commit 5b3ad94

File tree

6 files changed

+254
-32
lines changed

6 files changed

+254
-32
lines changed

‎mysql-test/main/func_math.result

Lines changed: 104 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1840,24 +1840,115 @@ CRC32(99999999999999999999999999999999)
18401840
SELECT CRC32(-99999999999999999999999999999999);
18411841
CRC32(-99999999999999999999999999999999)
18421842
1052326872
1843+
SELECT CRC32C(NULL), CRC32C(''), CRC32C('MariaDB'), CRC32C('mariadb');
1844+
CRC32C(NULL) CRC32C('') CRC32C('MariaDB') CRC32C('mariadb')
1845+
NULL 0 809606978 1378644259
1846+
SELECT CRC32(NULL,1),CRC32C(NULL,1), CRC32(1,''), CRC32C(1,'');
1847+
CRC32(NULL,1) CRC32C(NULL,1) CRC32(1,'') CRC32C(1,'')
1848+
NULL NULL 1 1
1849+
SELECT CRC32(42,''),CRC32C(42,''),CRC32('42',''),CRC32C('42','');
1850+
CRC32(42,'') CRC32C(42,'') CRC32('42','') CRC32C('42','')
1851+
42 42 42 42
1852+
SELECT CRC32(42,NULL),CRC32C(42,NULL);
1853+
CRC32(42,NULL) CRC32C(42,NULL)
1854+
NULL NULL
1855+
SELECT CRC32 ('5c',''),CRC32 ('5c',0),CRC32 ('5c', '0'),CRC32 ('5c',NULL);
1856+
CRC32 ('5c','') CRC32 ('5c',0) CRC32 ('5c', '0') CRC32 ('5c',NULL)
1857+
5 2226203566 2226203566 NULL
1858+
Warnings:
1859+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1860+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1861+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1862+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1863+
SELECT CRC32C('5c',''),CRC32C('5c',0),CRC32C('5c', '0'),CRC32C('5c',NULL);
1864+
CRC32C('5c','') CRC32C('5c',0) CRC32C('5c', '0') CRC32C('5c',NULL)
1865+
5 1466896124 1466896124 NULL
1866+
Warnings:
1867+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1868+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1869+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1870+
Warning 1292 Truncated incorrect INTEGER value: '5c'
1871+
SELECT CRC32('MariaDB',NULL),CRC32C('MariaDB',NULL);
1872+
CRC32('MariaDB',NULL) CRC32C('MariaDB',NULL)
1873+
NULL NULL
1874+
Warnings:
1875+
Warning 1292 Truncated incorrect INTEGER value: 'MariaDB'
1876+
Warning 1292 Truncated incorrect INTEGER value: 'MariaDB'
1877+
SELECT CRC32(CRC32('MySQL'),''),CRC32(CRC32('My'),'SQL'),CRC32(0,'MySQL');
1878+
CRC32(CRC32('MySQL'),'') CRC32(CRC32('My'),'SQL') CRC32(0,'MySQL')
1879+
3259397556 3259397556 3259397556
1880+
SELECT CRC32C(CRC32C('MariaDB'),''),CRC32C(CRC32C('Maria'),'DB'),CRC32C(0,'MariaDB');
1881+
CRC32C(CRC32C('MariaDB'),'') CRC32C(CRC32C('Maria'),'DB') CRC32C(0,'MariaDB')
1882+
809606978 809606978 809606978
1883+
select crc32(0,'My','SQL');
1884+
ERROR 42000: Incorrect parameter count in the call to native function 'crc32'
1885+
select crc32c(0,'Maria','DB');
1886+
ERROR 42000: Incorrect parameter count in the call to native function 'crc32c'
1887+
select crc32();
1888+
ERROR 42000: Incorrect parameter count in the call to native function 'crc32'
1889+
select crc32c();
1890+
ERROR 42000: Incorrect parameter count in the call to native function 'crc32c'
1891+
select crc32('' as empty);
1892+
ERROR 42000: Incorrect parameters in the call to native function 'crc32'
1893+
select crc32c('' as empty);
1894+
ERROR 42000: Incorrect parameters in the call to native function 'crc32c'
1895+
select crc32(0, '' as empty);
1896+
ERROR 42000: Incorrect parameters in the call to native function 'crc32'
1897+
select crc32c(0, '' as empty);
1898+
ERROR 42000: Incorrect parameters in the call to native function 'crc32c'
1899+
select crc32(0 as zero, '');
1900+
ERROR 42000: Incorrect parameters in the call to native function 'crc32'
1901+
select crc32c(0 as zero, '');
1902+
ERROR 42000: Incorrect parameters in the call to native function 'crc32c'
1903+
CREATE TEMPORARY TABLE t
1904+
(a CHAR(2), i INT UNSIGNED, c INT UNSIGNED AS (CRC32C(i,a)));
1905+
INSERT INTO t (a,i) VALUES ('DB',CRC32C('Maria'));
1906+
SELECT * FROM t;
1907+
a i c
1908+
DB 1253907744 809606978
1909+
DROP TEMPORARY TABLE t;
1910+
select crc32(4294967296,''), hex(char(4294967296));
1911+
crc32(4294967296,'') hex(char(4294967296))
1912+
0 00
1913+
select crc32(1e100,''), hex(char(1e100));
1914+
crc32(1e100,'') hex(char(1e100))
1915+
4294967295 FFFFFFFF
1916+
select crc32(10.11,''), hex(char(10.11));
1917+
crc32(10.11,'') hex(char(10.11))
1918+
10 0A
1919+
select crc32(-1,''), hex(char(-1));
1920+
crc32(-1,'') hex(char(-1))
1921+
4294967295 FFFFFFFF
1922+
select crc32('',''), hex(char(''));
1923+
crc32('','') hex(char(''))
1924+
0 00
1925+
Warnings:
1926+
Warning 1292 Truncated incorrect INTEGER value: ''
1927+
Warning 1292 Truncated incorrect INTEGER value: ''
1928+
select crc32(429496729656755555555555555555555555555555555555555555555555555555555555555555555555555,'a') as x;
1929+
x
1930+
3310005809
1931+
Warnings:
1932+
Warning 1916 Got overflow when converting '' to DECIMAL. Value truncated
1933+
Warning 1916 Got overflow when converting '99999999999999999999999999999999999999999999999999999999999999999' to INT. Value truncated
18431934
DROP TABLE IF EXISTS t;
18441935
Warnings:
18451936
Note 1051 Unknown table 'test.t'
18461937
CREATE TABLE t(a INT, b VARCHAR(2));
18471938
INSERT INTO t VALUES (1,'a'), (2,'qw'), (1,'t'), (3,'t');
1848-
SELECT crc32(SUM(a)) FROM t;
1849-
crc32(SUM(a))
1850-
1790921346
1851-
SELECT crc32(AVG(a)) FROM t GROUP BY b;
1852-
crc32(AVG(a))
1853-
768278432
1854-
2875100430
1855-
2875100430
1856-
SELECT crc32(MAX(b)) FROM t GROUP BY a;
1857-
crc32(MAX(b))
1858-
2238339752
1859-
3114057431
1860-
2238339752
1939+
SELECT crc32(SUM(a)),crc32c(SUM(a)) FROM t;
1940+
crc32(SUM(a)) crc32c(SUM(a))
1941+
1790921346 3058990603
1942+
SELECT crc32(AVG(a)),crc32c(AVG(a)) FROM t GROUP BY b;
1943+
crc32(AVG(a)) crc32c(AVG(a))
1944+
768278432 1816172052
1945+
2875100430 1492934094
1946+
2875100430 1492934094
1947+
SELECT crc32(MAX(b)),crc32c(MAX(b)) FROM t GROUP BY a;
1948+
crc32(MAX(b)) crc32c(MAX(b))
1949+
2238339752 3833565251
1950+
3114057431 4173859780
1951+
2238339752 3833565251
18611952
SELECT a, b, crc32(a) FROM t GROUP BY a,b HAVING crc32(MAX(a))=450215437;
18621953
a b crc32(a)
18631954
2 qw 450215437

‎mysql-test/main/func_math.test

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -849,15 +849,58 @@ SELECT CRC32('01234567'), CRC32('012345678');
849849
SELECT CRC32('~!@$%^*'), CRC32('-0.0001');
850850
SELECT CRC32(99999999999999999999999999999999);
851851
SELECT CRC32(-99999999999999999999999999999999);
852+
SELECT CRC32C(NULL), CRC32C(''), CRC32C('MariaDB'), CRC32C('mariadb');
853+
SELECT CRC32(NULL,1),CRC32C(NULL,1), CRC32(1,''), CRC32C(1,'');
854+
SELECT CRC32(42,''),CRC32C(42,''),CRC32('42',''),CRC32C('42','');
855+
SELECT CRC32(42,NULL),CRC32C(42,NULL);
856+
SELECT CRC32 ('5c',''),CRC32 ('5c',0),CRC32 ('5c', '0'),CRC32 ('5c',NULL);
857+
SELECT CRC32C('5c',''),CRC32C('5c',0),CRC32C('5c', '0'),CRC32C('5c',NULL);
858+
SELECT CRC32('MariaDB',NULL),CRC32C('MariaDB',NULL);
859+
SELECT CRC32(CRC32('MySQL'),''),CRC32(CRC32('My'),'SQL'),CRC32(0,'MySQL');
860+
SELECT CRC32C(CRC32C('MariaDB'),''),CRC32C(CRC32C('Maria'),'DB'),CRC32C(0,'MariaDB');
861+
862+
--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
863+
select crc32(0,'My','SQL');
864+
--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
865+
select crc32c(0,'Maria','DB');
866+
--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
867+
select crc32();
868+
--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT
869+
select crc32c();
870+
--error ER_WRONG_PARAMETERS_TO_NATIVE_FCT
871+
select crc32('' as empty);
872+
--error ER_WRONG_PARAMETERS_TO_NATIVE_FCT
873+
select crc32c('' as empty);
874+
--error ER_WRONG_PARAMETERS_TO_NATIVE_FCT
875+
select crc32(0, '' as empty);
876+
--error ER_WRONG_PARAMETERS_TO_NATIVE_FCT
877+
select crc32c(0, '' as empty);
878+
--error ER_WRONG_PARAMETERS_TO_NATIVE_FCT
879+
select crc32(0 as zero, '');
880+
--error ER_WRONG_PARAMETERS_TO_NATIVE_FCT
881+
select crc32c(0 as zero, '');
882+
883+
CREATE TEMPORARY TABLE t
884+
(a CHAR(2), i INT UNSIGNED, c INT UNSIGNED AS (CRC32C(i,a)));
885+
INSERT INTO t (a,i) VALUES ('DB',CRC32C('Maria'));
886+
SELECT * FROM t;
887+
DROP TEMPORARY TABLE t;
888+
889+
select crc32(4294967296,''), hex(char(4294967296));
890+
select crc32(1e100,''), hex(char(1e100));
891+
select crc32(10.11,''), hex(char(10.11));
892+
select crc32(-1,''), hex(char(-1));
893+
select crc32('',''), hex(char(''));
894+
select crc32(429496729656755555555555555555555555555555555555555555555555555555555555555555555555555,'a') as x;
852895

853896
# Test cases for using the function in aggregate functions, group-by, having
854897
# and order-by clauses
855898
DROP TABLE IF EXISTS t;
856899
CREATE TABLE t(a INT, b VARCHAR(2));
857900
INSERT INTO t VALUES (1,'a'), (2,'qw'), (1,'t'), (3,'t');
858-
SELECT crc32(SUM(a)) FROM t;
859-
SELECT crc32(AVG(a)) FROM t GROUP BY b;
860-
SELECT crc32(MAX(b)) FROM t GROUP BY a;
901+
SELECT crc32(SUM(a)),crc32c(SUM(a)) FROM t;
902+
SELECT crc32(AVG(a)),crc32c(AVG(a)) FROM t GROUP BY b;
903+
SELECT crc32(MAX(b)),crc32c(MAX(b)) FROM t GROUP BY a;
861904
SELECT a, b, crc32(a) FROM t GROUP BY a,b HAVING crc32(MAX(a))=450215437;
862905
SELECT a,b,concat(a,b),crc32(concat(a,b)) FROM t ORDER BY crc32(concat(a,b));
863906
DROP TABLE t;

‎mysys/crc32ieee.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* Copyright (c) 2020, 2021, MariaDB
1+
/* Copyright (c) 2020, 2022, MariaDB
22
33
This program is free software; you can redistribute it and/or modify
44
it under the terms of the GNU General Public License as published by
@@ -55,7 +55,7 @@ static const my_crc32_t my_checksum_func= init_crc32();
5555
# error "my_checksum() is defined in mysys/crc32/crc32_ppc64.c"
5656
#endif
5757
extern "C"
58-
unsigned int my_checksum(unsigned int crc, const void *data, size_t len)
58+
uint32 my_checksum(uint32 crc, const void *data, size_t len)
5959
{
6060
return my_checksum_func(crc, data, len);
6161
}

‎sql/item_create.cc

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
Copyright (c) 2000, 2011, Oracle and/or its affiliates.
3-
Copyright (c) 2008, 2021, MariaDB Corporation.
3+
Copyright (c) 2008, 2022, MariaDB Corporation.
44
55
This program is free software; you can redistribute it and/or modify
66
it under the terms of the GNU General Public License as published by
@@ -512,10 +512,10 @@ class Create_func_cot : public Create_func_arg1
512512
};
513513

514514

515-
class Create_func_crc32 : public Create_func_arg1
515+
class Create_func_crc32 : public Create_native_func
516516
{
517517
public:
518-
virtual Item *create_1_arg(THD *thd, Item *arg1);
518+
Item *create_native(THD *thd, LEX_CSTRING *, List<Item> *item_list) override;
519519

520520
static Create_func_crc32 s_singleton;
521521

@@ -525,6 +525,19 @@ class Create_func_crc32 : public Create_func_arg1
525525
};
526526

527527

528+
class Create_func_crc32c : public Create_native_func
529+
{
530+
public:
531+
Item *create_native(THD *thd, LEX_CSTRING *, List<Item> *item_list) override;
532+
533+
static Create_func_crc32c s_singleton;
534+
535+
protected:
536+
Create_func_crc32c() {}
537+
virtual ~Create_func_crc32c() {}
538+
};
539+
540+
528541
class Create_func_datediff : public Create_func_arg2
529542
{
530543
public:
@@ -3118,11 +3131,55 @@ Create_func_cot::create_1_arg(THD *thd, Item *arg1)
31183131
Create_func_crc32 Create_func_crc32::s_singleton;
31193132

31203133
Item*
3121-
Create_func_crc32::create_1_arg(THD *thd, Item *arg1)
3134+
Create_func_crc32::create_native(THD *thd, LEX_CSTRING *name,
3135+
List<Item> *item_list)
3136+
{
3137+
int argc= item_list ? item_list->elements : 0;
3138+
3139+
if (unlikely(argc != 1 && argc != 2))
3140+
{
3141+
my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name->str);
3142+
return nullptr;
3143+
}
3144+
3145+
Item *arg1= item_list->pop(), *arg2= argc < 2 ? nullptr : item_list->pop();
3146+
3147+
/* This was checked in Create_native_func::create_func() */
3148+
DBUG_ASSERT(!arg1->is_explicit_name());
3149+
DBUG_ASSERT(!arg2 || !arg2->is_explicit_name());
3150+
3151+
return arg2
3152+
? new (thd->mem_root) Item_func_crc32(thd, false, arg1, arg2)
3153+
: new (thd->mem_root) Item_func_crc32(thd, false, arg1);
3154+
}
3155+
3156+
3157+
Create_func_crc32c Create_func_crc32c::s_singleton;
3158+
3159+
Item*
3160+
Create_func_crc32c::create_native(THD *thd, LEX_CSTRING *name,
3161+
List<Item> *item_list)
31223162
{
3123-
return new (thd->mem_root) Item_func_crc32(thd, arg1);
3163+
int argc= item_list ? item_list->elements : 0;
3164+
3165+
if (unlikely(argc != 1 && argc != 2))
3166+
{
3167+
my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name->str);
3168+
return nullptr;
3169+
}
3170+
3171+
Item *arg1= item_list->pop(), *arg2= argc < 2 ? nullptr : item_list->pop();
3172+
3173+
/* This was checked in Create_native_func::create_func() */
3174+
DBUG_ASSERT(!arg1->is_explicit_name());
3175+
DBUG_ASSERT(!arg2 || !arg2->is_explicit_name());
3176+
3177+
return arg2
3178+
? new (thd->mem_root) Item_func_crc32(thd, true, arg1, arg2)
3179+
: new (thd->mem_root) Item_func_crc32(thd, true, arg1);
31243180
}
31253181

3182+
31263183
Create_func_datediff Create_func_datediff::s_singleton;
31273184

31283185
Item*
@@ -5555,6 +5612,7 @@ Native_func_registry func_array[] =
55555612
{ { STRING_WITH_LEN("COS") }, BUILDER(Create_func_cos)},
55565613
{ { STRING_WITH_LEN("COT") }, BUILDER(Create_func_cot)},
55575614
{ { STRING_WITH_LEN("CRC32") }, BUILDER(Create_func_crc32)},
5615+
{ { STRING_WITH_LEN("CRC32C") }, BUILDER(Create_func_crc32c)},
55585616
{ { STRING_WITH_LEN("DATEDIFF") }, BUILDER(Create_func_datediff)},
55595617
{ { STRING_WITH_LEN("DAYNAME") }, BUILDER(Create_func_dayname)},
55605618
{ { STRING_WITH_LEN("DAYOFMONTH") }, BUILDER(Create_func_dayofmonth)},

‎sql/item_strfunc.cc

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
Copyright (c) 2000, 2017, Oracle and/or its affiliates.
3-
Copyright (c) 2009, 2021, MariaDB Corporation.
3+
Copyright (c) 2009, 2022, MariaDB Corporation.
44
55
This program is free software; you can redistribute it and/or modify
66
it under the terms of the GNU General Public License as published by
@@ -4385,14 +4385,32 @@ longlong Item_func_uncompressed_length::val_int()
43854385
longlong Item_func_crc32::val_int()
43864386
{
43874387
DBUG_ASSERT(fixed());
4388-
String *res=args[0]->val_str(&value);
4388+
DBUG_ASSERT(arg_count == 1 || arg_count == 2);
4389+
String *res;
4390+
longlong crc;
4391+
if (arg_count > 1)
4392+
{
4393+
crc= args[0]->val_int();
4394+
null_value= args[0]->null_value;
4395+
if (null_value)
4396+
return 0;
4397+
res= args[1]->val_str(&value);
4398+
}
4399+
else
4400+
{
4401+
crc= 0;
4402+
null_value= 0;
4403+
res= args[0]->val_str(&value);
4404+
}
4405+
43894406
if (!res)
43904407
{
43914408
null_value=1;
43924409
return 0; /* purecov: inspected */
43934410
}
4394-
null_value=0;
4395-
return (longlong) my_checksum(0L, (uchar*)res->ptr(), res->length());
4411+
4412+
return static_cast<longlong>
4413+
(ulonglong{crc_func(uint32_t(crc), res->ptr(), res->length())});
43964414
}
43974415

43984416
#ifdef HAVE_COMPRESS

‎sql/item_strfunc.h

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
/*
55
Copyright (c) 2000, 2011, Oracle and/or its affiliates.
6-
Copyright (c) 2009, 2019, MariaDB
6+
Copyright (c) 2009, 2022, MariaDB
77
88
This program is free software; you can redistribute it and/or modify
99
it under the terms of the GNU General Public License as published by
@@ -1945,15 +1945,27 @@ class Item_func_weight_string :public Item_str_func
19451945
class Item_func_crc32 :public Item_long_func
19461946
{
19471947
bool check_arguments() const override
1948-
{ return args[0]->check_type_can_return_str(func_name_cstring()); }
1948+
{
1949+
return args[0]->check_type_can_return_str(func_name_cstring()) &&
1950+
(arg_count == 1 ||
1951+
args[1]->check_type_can_return_int(func_name_cstring()));
1952+
}
19491953
String value;
1954+
uint32 (*const crc_func)(uint32, const void*, size_t);
19501955
public:
1951-
Item_func_crc32(THD *thd, Item *a): Item_long_func(thd, a)
1956+
Item_func_crc32(THD *thd, bool Castagnoli, Item *a) :
1957+
Item_long_func(thd, a),
1958+
crc_func(Castagnoli ? my_crc32c : my_checksum)
1959+
{ unsigned_flag= 1; }
1960+
Item_func_crc32(THD *thd, bool Castagnoli, Item *a, Item *b) :
1961+
Item_long_func(thd, a, b),
1962+
crc_func(Castagnoli ? my_crc32c : my_checksum)
19521963
{ unsigned_flag= 1; }
19531964
LEX_CSTRING func_name_cstring() const override
19541965
{
1955-
static LEX_CSTRING name= {STRING_WITH_LEN("crc32") };
1956-
return name;
1966+
static LEX_CSTRING crc32_name= {STRING_WITH_LEN("crc32") };
1967+
static LEX_CSTRING crc32c_name= {STRING_WITH_LEN("crc32c") };
1968+
return crc_func == my_crc32c ? crc32c_name : crc32_name;
19571969
}
19581970
bool fix_length_and_dec() override { max_length=10; return FALSE; }
19591971
longlong val_int() override;

0 commit comments

Comments
 (0)