Skip to content

Commit 62d73df

Browse files
author
Varun Gupta
committed
MDEV-19232: Floating point precision / value comparison problem
The issue occurs when the subquery_cache is enabled. When there is a cache miss the division was leading to a value with scale 9. In the case of cache hit the value returned was of scale 9 and due to the different values for the scales the where condition evaluated to FALSE, hence the output was incomplete. To fix this problem we need to round up the decimal to the limit mentioned in Item::decimals. This would make sure the values are compared with the same scale.
1 parent 57ec42b commit 62d73df

File tree

11 files changed

+68
-30
lines changed

11 files changed

+68
-30
lines changed

‎mysql-test/r/func_group.result

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1186,13 +1186,13 @@ i count(*) std(e1/e2)
11861186
3 4 0.00000000
11871187
select std(s1/s2) from bug22555;
11881188
std(s1/s2)
1189-
0.21325764
1189+
0.21328517
11901190
select std(o1/o2) from bug22555;
11911191
std(o1/o2)
11921192
0.2132576358664934
11931193
select std(e1/e2) from bug22555;
11941194
std(e1/e2)
1195-
0.21325764
1195+
0.21328517
11961196
set @saved_div_precision_increment=@@div_precision_increment;
11971197
set div_precision_increment=19;
11981198
select i, count(*), std(s1/s2) from bug22555 group by i order by i;

‎mysql-test/r/parser_precedence.result

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,7 @@ select 4 - 3 * 2, (4 - 3) * 2, 4 - (3 * 2);
619619
Testing that / is left associative
620620
select 15 / 5 / 3, (15 / 5) / 3, 15 / (5 / 3);
621621
15 / 5 / 3 (15 / 5) / 3 15 / (5 / 3)
622-
1.00000000 1.00000000 9.0000
622+
1.00000000 1.00000000 8.9998
623623
Testing that / has precedence over |
624624
select 105 / 5 | 2, (105 / 5) | 2, 105 / (5 | 2);
625625
105 / 5 | 2 (105 / 5) | 2 105 / (5 | 2)

‎mysql-test/r/subselect4.result

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2584,3 +2584,27 @@ l1 i2
25842584
e 2
25852585
o 6
25862586
DROP TABLE t1, t2;
2587+
#
2588+
# MDEV-19232: Floating point precision / value comparison problem
2589+
#
2590+
CREATE TABLE t1 (region varchar(60), area decimal(10,0), population decimal(11,0));
2591+
INSERT INTO t1 VALUES ('Central America and the Caribbean',91,11797);
2592+
INSERT INTO t1 VALUES ('Central America and the Caribbean',442,66422);
2593+
SET @save_optimizer_switch=@@optimizer_switch;
2594+
SET optimizer_switch='subquery_cache=on';
2595+
SELECT
2596+
population, area, population/area,
2597+
cast(population/area as DECIMAL(20,9)) FROM t1 LIMIT 1;
2598+
population area population/area cast(population/area as DECIMAL(20,9))
2599+
11797 91 129.6374 129.637400000
2600+
SELECT * FROM t1 A
2601+
WHERE population/area = (SELECT MAX(population/area) from t1 B where A.region = B.region);
2602+
region area population
2603+
Central America and the Caribbean 442 66422
2604+
SET optimizer_switch='subquery_cache=off';
2605+
SELECT * FROM t1 A
2606+
WHERE population/area = (SELECT MAX(population/area) from t1 B where A.region = B.region);
2607+
region area population
2608+
Central America and the Caribbean 442 66422
2609+
SET @@optimizer_switch= @save_optimizer_switch;
2610+
DROP TABLE t1;

‎mysql-test/r/type_newdecimal.result

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1530,11 +1530,8 @@ select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *
15301530
1.01500000 * 1.01500000 * 0.99500000)
15311531
0.812988073953673124592306939480
15321532
create table t1 as select 5.05 / 0.014;
1533-
Warnings:
1534-
Note 1265 Data truncated for column '5.05 / 0.014' at row 1
15351533
show warnings;
15361534
Level Code Message
1537-
Note 1265 Data truncated for column '5.05 / 0.014' at row 1
15381535
show create table t1;
15391536
Table Create Table
15401537
t1 CREATE TABLE `t1` (
@@ -1649,8 +1646,6 @@ my_col
16491646
0.123456789123456789123456789123
16501647
DROP TABLE t1;
16511648
CREATE TABLE t1 SELECT 1 / .123456789123456789123456789123456789123456789123456789123456789123456789123456789 AS my_col;
1652-
Warnings:
1653-
Note 1265 Data truncated for column 'my_col' at row 1
16541649
DESCRIBE t1;
16551650
Field Type Null Key Default Extra
16561651
my_col decimal(65,4) YES NULL

‎mysql-test/r/type_ranges.result

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,6 @@ DROP INDEX test ON t1;
9494
insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one');
9595
insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one');
9696
insert into t1 values (0,1/3,3,3,3,3,3,3,3,3,3,3,3,3,NULL,'19970303','10:10:10','19970303101010','','','','3',3,3);
97-
Warnings:
98-
Warning 1265 Data truncated for column 'string' at row 1
9997
insert into t1 values (0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,NULL,19970807,080706,19970403090807,-1,-1,-1,'-1',-1,-1);
10098
Warnings:
10199
Warning 1264 Out of range value for column 'utiny' at row 1
@@ -136,7 +134,7 @@ select auto,string,tiny,short,medium,long_int,longlong,real_float,real_double,ut
136134
auto string tiny short medium long_int longlong real_float real_double utiny ushort umedium ulong ulonglong mod(floor(time_stamp/1000000),1000000)-mod(curdate(),1000000) date_field time_field date_time blob_col tinyblob_col mediumblob_col longblob_col
137135
10 1 1 1 1 1 1 1.0 1.0000 1 00001 1 1 1 0 0000-00-00 00:00:00 0000-00-00 00:00:00 1 1 1 1
138136
11 2 2 2 2 2 2 2.0 2.0000 2 00002 2 2 2 0 NULL NULL NULL NULL NULL 2 2
139-
12 0.33333333 3 3 3 3 3 3.0 3.0000 3 00003 3 3 3 0 1997-03-03 10:10:10 1997-03-03 10:10:10 3
137+
12 0.3333 3 3 3 3 3 3.0 3.0000 3 00003 3 3 3 0 1997-03-03 10:10:10 1997-03-03 10:10:10 3
140138
13 -1 -1 -1 -1 -1 -1 -1.0 -1.0000 0 00000 0 0 0 0 1997-08-07 08:07:06 1997-04-03 09:08:07 -1 -1 -1 -1
141139
14 -429496729 -128 -32768 -8388608 -2147483648 -4294967295 -4294967296.0 -4294967295.0000 0 00000 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 -4294967295 -4294967295 -4294967295 -4294967295
142140
15 4294967295 127 32767 8388607 2147483647 4294967295 4294967296.0 4294967295.0000 255 65535 16777215 4294967295 4294967295 0 0000-00-00 00:00:00 0000-00-00 00:00:00 4294967295 4294967295 4294967295 4294967295
@@ -188,7 +186,7 @@ Warning 1265 Data truncated for column 'new_field' at row 7
188186
select * from t2;
189187
auto string mediumblob_col new_field
190188
1 2 2 ne
191-
2 0.33333333 ne
189+
2 0.3333 ne
192190
3 -1 -1 ne
193191
4 -429496729 -4294967295 ne
194192
5 4294967295 4294967295 ne

‎mysql-test/suite/sys_vars/r/div_precision_increment_func.result

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ INSERT into t1(name, salary, income_tax) values('Record_2', 501, 501*2.5/1000);
5050
INSERT into t1(name, salary, income_tax) values('Record_3', 210, 210*2.5/1000);
5151
SELECT * from t1;
5252
id name salary income_tax
53-
1 Record_1 100011 250.027
54-
2 Record_2 501 1.2525
55-
3 Record_3 210 0.525
53+
1 Record_1 100011 250.03
54+
2 Record_2 501 1.25
55+
3 Record_3 210 0.53
5656
## Creating new connection ##
5757
## Verifying session & global value of variable ##
5858
SELECT @@global.div_precision_increment = 2;
@@ -67,11 +67,11 @@ INSERT into t1(name, salary, income_tax) values('Record_5', 501, 501*2.5/1000);
6767
INSERT into t1(name, salary, income_tax) values('Record_6', 210, 210*2.5/1000);
6868
SELECT * from t1;
6969
id name salary income_tax
70-
1 Record_1 100011 250.027
71-
2 Record_2 501 1.2525
72-
3 Record_3 210 0.525
73-
4 Record_4 100011 250.027
74-
5 Record_5 501 1.2525
70+
1 Record_1 100011 250.03
71+
2 Record_2 501 1.25
72+
3 Record_3 210 0.53
73+
4 Record_4 100011 250.028
74+
5 Record_5 501 1.253
7575
6 Record_6 210 0.525
7676
## Dropping table t1 ##
7777
drop table t1;

‎mysql-test/suite/vcol/r/not_supported.result

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ a b v
5151
flush tables;
5252
select * from t1;
5353
a b v
54-
1 2 0.3333333330000000000
54+
1 2 0.3333000000000000000
5555
select * from t5;
5656
a b v
5757
20141010 2 October

‎mysql-test/t/subselect4.test

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2112,3 +2112,29 @@ INSERT INTO t2 VALUES ('k'),('rid'),('f'),('x');
21122112
EXPLAIN EXTENDED SELECT * FROM t1 where ( t1.l1 < ANY (SELECT MAX(t2.v1) FROM t2));
21132113
SELECT * FROM t1 where ( t1.l1 < ANY (SELECT MAX(t2.v1) FROM t2));
21142114
DROP TABLE t1, t2;
2115+
2116+
--echo #
2117+
--echo # MDEV-19232: Floating point precision / value comparison problem
2118+
--echo #
2119+
2120+
CREATE TABLE t1 (region varchar(60), area decimal(10,0), population decimal(11,0));
2121+
INSERT INTO t1 VALUES ('Central America and the Caribbean',91,11797);
2122+
INSERT INTO t1 VALUES ('Central America and the Caribbean',442,66422);
2123+
2124+
SET @save_optimizer_switch=@@optimizer_switch;
2125+
SET optimizer_switch='subquery_cache=on';
2126+
2127+
SELECT
2128+
population, area, population/area,
2129+
cast(population/area as DECIMAL(20,9)) FROM t1 LIMIT 1;
2130+
2131+
SELECT * FROM t1 A
2132+
WHERE population/area = (SELECT MAX(population/area) from t1 B where A.region = B.region);
2133+
2134+
SET optimizer_switch='subquery_cache=off';
2135+
SELECT * FROM t1 A
2136+
WHERE population/area = (SELECT MAX(population/area) from t1 B where A.region = B.region);
2137+
2138+
SET @@optimizer_switch= @save_optimizer_switch;
2139+
2140+
DROP TABLE t1;

‎sql/item_func.cc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1719,6 +1719,8 @@ my_decimal *Item_func_div::decimal_op(my_decimal *decimal_value)
17191719
null_value= 1;
17201720
return 0;
17211721
}
1722+
my_decimal_round(E_DEC_FATAL_ERROR, decimal_value,
1723+
decimals, FALSE, decimal_value);
17221724
return decimal_value;
17231725
}
17241726

‎storage/tokudb/mysql-test/tokudb/r/type_newdecimal.result

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1531,11 +1531,8 @@ select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 *
15311531
1.01500000 * 1.01500000 * 0.99500000)
15321532
0.812988073953673124592306939480
15331533
create table t1 as select 5.05 / 0.014;
1534-
Warnings:
1535-
Note 1265 Data truncated for column '5.05 / 0.014' at row 1
15361534
show warnings;
15371535
Level Code Message
1538-
Note 1265 Data truncated for column '5.05 / 0.014' at row 1
15391536
show create table t1;
15401537
Table Create Table
15411538
t1 CREATE TABLE `t1` (
@@ -1650,8 +1647,6 @@ my_col
16501647
0.123456789123456789123456789123
16511648
DROP TABLE t1;
16521649
CREATE TABLE t1 SELECT 1 / .123456789123456789123456789123456789123456789123456789123456789123456789123456789 AS my_col;
1653-
Warnings:
1654-
Note 1265 Data truncated for column 'my_col' at row 1
16551650
DESCRIBE t1;
16561651
Field Type Null Key Default Extra
16571652
my_col decimal(65,4) YES NULL

0 commit comments

Comments
 (0)