From a33108add842cf5cb08c3d9ee070582e244f6c72 Mon Sep 17 00:00:00 2001
From: Vladislav Shpilevoy <v.shpilevoy@tarantool.org>
Date: Mon, 1 Jun 2020 20:49:11 +0200
Subject: [PATCH] sql: fix mem_apply_type double type truncation

mem_apply_type(), when tried to cast a double value to an integer,
used the expressions:

    int64_t i = (int64_t) d;
    uint64_t u = (uint64_t) d;

To obtain integer versions of the double value, cast them back to
double, and see if they are equal. Assuming that if they are, the
double can be safely cast to one of them.

But this is undefined behaviour. Double can't be cast to int64_t,
if it is > INT64_MAX or < INT64_MIN. And can't be cast to
uint64_t, if it is < 0 or > UINT64_MAX.

The patch adds explicit checks for these borders before doing the
cast.

Part of #4609
---
 src/box/sql/vdbe.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/box/sql/vdbe.c b/src/box/sql/vdbe.c
index 5bc106b5dd3b..6b769805ceba 100644
--- a/src/box/sql/vdbe.c
+++ b/src/box/sql/vdbe.c
@@ -324,12 +324,18 @@ mem_apply_type(struct Mem *record, enum field_type type)
 			return 0;
 		if ((record->flags & MEM_Real) == MEM_Real) {
 			double d = record->u.r;
-			int64_t i = (int64_t) d;
-			uint64_t u = (uint64_t) d;
-			if (i == d)
-				mem_set_int(record, i, i <= -1);
-			else if (u == d)
-				mem_set_u64(record, u);
+			if (d >= 0) {
+				if (double_compare_uint64(d, UINT64_MAX,
+							  1) > 0)
+					return 0;
+				if ((double)(uint64_t)d == d)
+					mem_set_u64(record, (uint64_t)d);
+			} else {
+				if (double_compare_nint64(d, INT64_MIN, 1) < 0)
+					return 0;
+				if ((double)(int64_t)d == d)
+					mem_set_int(record, (int64_t)d, true);
+			}
 			return 0;
 		}
 		if ((record->flags & MEM_Str) != 0) {