sql: fix mem_apply_type double type truncation

mem_apply_type(), when tried to cast a double value to an integer,
used the expressions:

    int64_t i = (int64_t) d;
    uint64_t u = (uint64_t) d;

To obtain integer versions of the double value, cast them back to
double, and see if they are equal. Assuming that if they are, the
double can be safely cast to one of them.

But this is undefined behaviour. Double can't be cast to int64_t,
if it is > INT64_MAX or < INT64_MIN. And can't be cast to
uint64_t, if it is < 0 or > UINT64_MAX.

The patch adds explicit checks for these borders before doing the
cast.

Part of #4609

src/box/sql/vdbe.c

@@ -324,12 +324,18 @@ mem_apply_type(struct Mem *record, enum field_type type)
 			return 0;
 		if ((record->flags & MEM_Real) == MEM_Real) {
 			double d = record->u.r;
-			int64_t i = (int64_t) d;
-			uint64_t u = (uint64_t) d;
-			if (i == d)
-				mem_set_int(record, i, i <= -1);
-			else if (u == d)
-				mem_set_u64(record, u);
+			if (d >= 0) {
+				if (double_compare_uint64(d, UINT64_MAX,
+							  1) > 0)
+					return 0;
+				if ((double)(uint64_t)d == d)
+					mem_set_u64(record, (uint64_t)d);
+			} else {
+				if (double_compare_nint64(d, INT64_MIN, 1) < 0)
+					return 0;
+				if ((double)(int64_t)d == d)
+					mem_set_int(record, (int64_t)d, true);
+			}
 			return 0;
 		}
 		if ((record->flags & MEM_Str) != 0) {