trying to stabilize floating-point tests

Author: vuvova

mysql-test/main/vector.result

@@ -108,14 +108,14 @@ id	hex(v)	vec_totext(v)
 flush tables;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 select id,vec_distance_euclidean(x'b047263C9f87233fcfd27e3eae493e3f0329f43e', v) d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 select id>0,vec_distance_euclidean(v, NULL) d from t1 order by d limit 3;
 id>0	d
 1	NULL
@@ -138,156 +138,156 @@ id1	id2	vec_distance_euclidean(t1.v, t2.v)
 8	8	0
 9	9	0
 10	10	0
-7	10	0.35209010323904116
-10	7	0.35209010323904116
-1	7	0.557267332724855
-7	1	0.557267332724855
-2	3	0.6065128837978769
-3	2	0.6065128837978769
-1	3	0.6128238020507096
-3	1	0.6128238020507096
-5	8	0.6219995745138945
-8	5	0.6219995745138945
-3	10	0.6523185662547816
-10	3	0.6523185662547816
-9	10	0.6732681362788765
-10	9	0.6732681362788765
-3	7	0.6799892416547949
-7	3	0.6799892416547949
-3	9	0.6820752294088018
-9	3	0.6820752294088018
-2	10	0.6916305331777215
-10	2	0.6916305331777215
-2	9	0.6966650510789955
-9	2	0.6966650510789955
-3	6	0.7102823580937639
-6	3	0.7102823580937639
-2	7	0.7120217580666971
-7	2	0.7120217580666971
-2	6	0.7351618106552689
-6	2	0.7351618106552689
-1	10	0.7386864491588024
-10	1	0.7386864491588024
-4	6	0.7784357824370262
-6	4	0.7784357824370262
-4	8	0.7795837407361241
-8	4	0.7795837407361241
-4	5	0.8132007346697969
-5	4	0.8132007346697969
-2	4	0.8260925223296488
-4	2	0.8260925223296488
-5	10	0.8286488932765299
-10	5	0.8286488932765299
-5	9	0.8769351333060768
-9	5	0.8769351333060768
-1	6	0.8861410875047832
-6	1	0.8861410875047832
-3	5	0.9224201772876247
-5	3	0.9224201772876247
-4	7	0.9347916246876117
-7	4	0.9347916246876117
-7	9	0.9364253407685257
-9	7	0.9364253407685257
-3	4	0.9757105842688992
-4	3	0.9757105842688992
-1	2	0.9810272439433514
-2	1	0.9810272439433514
-1	4	0.9965475544626712
-4	1	0.9965475544626712
-5	7	0.9976863778073342
-7	5	0.9976863778073342
-4	10	1.0109345944029724
-10	4	1.0109345944029724
-1	5	1.0208359400987237
-5	1	1.0208359400987237
-6	7	1.0221332668982412
-7	6	1.0221332668982412
-2	5	1.050769316594881
-5	2	1.050769316594881
-6	8	1.103420381318026
-8	6	1.103420381318026
-3	8	1.1170300826294572
-8	3	1.1170300826294572
-6	10	1.1523451990991307
-10	6	1.1523451990991307
-1	9	1.1637750565139302
-9	1	1.1637750565139302
-2	8	1.1736571017573874
-8	2	1.1736571017573874
-4	9	1.1746893942711878
-9	4	1.1746893942711878
-1	8	1.1909959973982214
-8	1	1.1909959973982214
-8	10	1.209359617652948
-10	8	1.209359617652948
-6	9	1.214529873940304
-9	6	1.214529873940304
-5	6	1.227278506501395
-6	5	1.227278506501395
-8	9	1.2575258643523053
-9	8	1.2575258643523053
-7	8	1.288239696195716
-8	7	1.288239696195716
+7	10	0.35209
+10	7	0.35209
+1	7	0.55726
+7	1	0.55726
+2	3	0.60651
+3	2	0.60651
+1	3	0.61282
+3	1	0.61282
+5	8	0.62199
+8	5	0.62199
+3	10	0.65231
+10	3	0.65231
+9	10	0.67326
+10	9	0.67326
+3	7	0.67998
+7	3	0.67998
+3	9	0.68207
+9	3	0.68207
+2	10	0.69163
+10	2	0.69163
+2	9	0.69666
+9	2	0.69666
+3	6	0.71028
+6	3	0.71028
+2	7	0.71202
+7	2	0.71202
+2	6	0.73516
+6	2	0.73516
+1	10	0.73868
+10	1	0.73868
+4	6	0.77843
+6	4	0.77843
+4	8	0.77958
+8	4	0.77958
+4	5	0.81320
+5	4	0.81320
+2	4	0.82609
+4	2	0.82609
+5	10	0.82864
+10	5	0.82864
+5	9	0.87693
+9	5	0.87693
+1	6	0.88614
+6	1	0.88614
+3	5	0.92242
+5	3	0.92242
+4	7	0.93479
+7	4	0.93479
+7	9	0.93642
+9	7	0.93642
+3	4	0.97571
+4	3	0.97571
+1	2	0.98102
+2	1	0.98102
+1	4	0.99654
+4	1	0.99654
+5	7	0.99768
+7	5	0.99768
+4	10	1.01093
+10	4	1.01093
+1	5	1.02083
+5	1	1.02083
+6	7	1.02213
+7	6	1.02213
+2	5	1.05076
+5	2	1.05076
+6	8	1.10342
+8	6	1.10342
+3	8	1.11703
+8	3	1.11703
+6	10	1.15234
+10	6	1.15234
+1	9	1.16377
+9	1	1.16377
+2	8	1.17365
+8	2	1.17365
+4	9	1.17468
+9	4	1.17468
+1	8	1.19099
+8	1	1.19099
+8	10	1.20935
+10	8	1.20935
+6	9	1.21452
+9	6	1.21452
+5	6	1.22727
+6	5	1.22727
+8	9	1.25752
+9	8	1.25752
+7	8	1.28823
+8	7	1.28823
 select * from (
 select id,vec_distance_euclidean(v, x'b047263C9F87233fcfd27e3eae493e3f0329f43e') d
 from t1 where id < 10
 ) u order by d limit 3;
 id	d
-9	0.4719976290006591
-3	0.5865673124650332
+9	0.47199
+3	0.58656
 flush session status;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	0
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 flush session status;
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 delete from t1 where v = x'7b713f3e5258323f80d1113d673b2b3f66e3583f';
 select id,vec_distance_euclidean(v, x'B047263C9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.5069011044450041
-3	0.5865673124650332
-7	0.7344464697214867
+10	0.50690
+3	0.58656
+7	0.73444
 insert t1 (v) values (x'7b713f3e5258323f80d1113d673b2b3f66e3583f');
 select id,vec_distance_euclidean(v, x'b047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-11	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+11	0.47199
+10	0.50690
+3	0.58656
 select id,vec_distance_euclidean(v, x'B047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-11	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
-7	0.7344464697214867
-5	0.7671033529042712
+11	0.47199
+10	0.50690
+3	0.58656
+7	0.73444
+5	0.76710
 update t1 set v=x'76EDFC3E4B57243F10F8423FB158713F020BAA3E' where v=x'6CA1D43E9DF91B3FE580DA3E1C247D3F147CF33E';
 select id,vec_distance_euclidean(v, x'B047263C9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-11	0.4719976290006591
-3	0.5865673124650332
-7	0.7344464697214867
-10	0.746836719209219
-5	0.7671033529042712
+11	0.47199
+3	0.58656
+7	0.73444
+10	0.74683
+5	0.76710
 delete from t1;
 insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'),
@@ -301,11 +301,11 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 select id,vec_distance_euclidean(v, x'b047263c9f87233Fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-20	0.4719976290006591
-21	0.5069011044450041
-14	0.5865673124650332
-18	0.7344464697214867
-16	0.7671033529042712
+20	0.47199
+21	0.50690
+14	0.58656
+18	0.73444
+16	0.76710
 insert t1 (v) values ('');
 ERROR 22007: Incorrect vector value: '...' for column `test`.`t1`.`v` at row 1
 insert t1 (v) values (x'1234');
@@ -436,32 +436,32 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 flush session status;
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	0
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 flush session status;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11