Permalink
Browse files

compact IEEE 754 double encoding in external binary format for ei

Implement the compact IEEE 754 double encoding in external binary
format for ei. Encoding for ei now always produces the NEW_FLOAT_EXT
format. Decoding and term printing handle both the old ERL_FLOAT_EXT
encoding and the new NEW_FLOAT_EXT encoding. Legacy erl_interface code
also handles the new encoding, but still produces the ERL_FLOAT_EXT
encoding by default.

Also enable the DFLAG_NEW_FLOATS distribution flag.

Reduce the number of copies of the code for encoding and decoding
doubles throughout ei and erl_interface by instead calling the ei
encoding and decoding functions wherever possible.

Restore commented-out float tests in ei_decode_SUITE and
ei_encode_SUITE in lib/erl_interface/test. Modify them to make them
match the style of other tests in the same suites.

These changes are based on an ei float patch from Serge Aleynikov
originally submitted against R12B-2 in July 2008.
  • Loading branch information...
1 parent 3b3bae9 commit f9eeacca3bc38833e4f86764d40345506f86e09d @vinoski committed May 29, 2010
@@ -110,6 +110,7 @@
#define ERL_SMALL_INTEGER_EXT 'a'
#define ERL_INTEGER_EXT 'b'
#define ERL_FLOAT_EXT 'c'
+#define NEW_FLOAT_EXT 'F'
#define ERL_ATOM_EXT 'd'
#define ERL_REFERENCE_EXT 'e'
#define ERL_NEW_REFERENCE_EXT 'r'
@@ -1323,7 +1323,8 @@ static int send_name_or_challenge(int fd, char *nodename,
put32be(s, (DFLAG_EXTENDED_REFERENCES
| DFLAG_EXTENDED_PIDS_PORTS
| DFLAG_FUN_TAGS
- | DFLAG_NEW_FUN_TAGS));
+ | DFLAG_NEW_FUN_TAGS
+ | DFLAG_NEW_FLOATS));
if (f_chall)
put32be(s, challenge);
memcpy(s, nodename, strlen(nodename));
@@ -1393,6 +1394,11 @@ static int recv_challenge(int fd, unsigned *challenge,
goto error;
}
+ if (!(*flags & DFLAG_NEW_FLOATS)) {
+ EI_TRACE_ERR0("recv_challenge","<- RECV_CHALLENGE peer cannot "
+ "handle binary float encoding");
+ goto error;
+ }
if (getpeername(fd, (struct sockaddr *) &sin, &sin_len) < 0) {
EI_TRACE_ERR0("recv_challenge","<- RECV_CHALLENGE can't get peername");
@@ -101,6 +101,7 @@ extern int h_errno;
#define DFLAG_FUN_TAGS 16
#define DFLAG_NEW_FUN_TAGS 0x80
#define DFLAG_EXTENDED_PIDS_PORTS 0x100
+#define DFLAG_NEW_FLOATS 0x800
ei_cnode *ei_fd_to_cnode(int fd);
int ei_distversion(int fd);
@@ -26,14 +26,22 @@ int ei_decode_double(const char *buf, int *index, double *p)
{
const char *s = buf + *index;
const char *s0 = s;
- double f;
+ FloatExt f;
- if (get8(s) != ERL_FLOAT_EXT) return -1;
-
- if (sscanf(s, "%lf", &f) != 1) return -1;
+ switch (get8(s)) {
+ case ERL_FLOAT_EXT:
+ if (sscanf(s, "%lf", &f.d) != 1) return -1;
+ s += 31;
+ break;
+ case NEW_FLOAT_EXT:
+ /* IEEE 754 format */
+ f.val = get64be(s);
+ break;
+ default:
+ return -1;
+ }
- s += 31;
- if (p) *p = f;
+ if (p) *p = f.d;
*index += s-s0;
return 0;
}
@@ -77,6 +77,7 @@ int ei_skip_term(const char* buf, int* index)
if (ei_decode_big(buf, index, NULL) < 0) return -1;
break;
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
if (ei_decode_double(buf, index, NULL) < 0) return -1;
break;
case ERL_FUN_EXT:
@@ -27,13 +27,13 @@ int ei_encode_double(char *buf, int *index, double p)
char *s = buf + *index;
char *s0 = s;
- if (!buf) s ++;
+ if (!buf)
+ s += 9;
else {
- put8(s,ERL_FLOAT_EXT);
- memset(s, 0, 31);
- sprintf(s, "%.20e", p);
+ /* IEEE 754 format */
+ put8(s, NEW_FLOAT_EXT);
+ put64be(s, ((FloatExt*)&p)->val);
}
- s += 31;
*index += s-s0;
@@ -59,6 +59,7 @@ int ei_decode_term(const char *buf, int *index, void *t)
return ei_decode_long(buf,index,NULL);
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
return ei_decode_double(buf,index,NULL);
case ERL_ATOM_EXT:
@@ -102,6 +102,7 @@ void erl_init_marshal(void)
cmp_array[ERL_SMALL_INTEGER_EXT] = 1;
cmp_array[ERL_INTEGER_EXT] = 1;
cmp_array[ERL_FLOAT_EXT] = 1;
+ cmp_array[NEW_FLOAT_EXT] = 1;
cmp_array[ERL_SMALL_BIG_EXT] = 1;
cmp_array[ERL_LARGE_BIG_EXT] = 1;
cmp_array[ERL_ATOM_EXT] = 2;
@@ -124,6 +125,7 @@ void erl_init_marshal(void)
cmp_num_class[ERL_SMALL_INTEGER_EXT] = SMALL;
cmp_num_class[ERL_INTEGER_EXT] = SMALL;
cmp_num_class[ERL_FLOAT_EXT] = FLOAT;
+ cmp_num_class[NEW_FLOAT_EXT] = FLOAT;
cmp_num_class[ERL_SMALL_BIG_EXT] = BIG;
cmp_num_class[ERL_LARGE_BIG_EXT] = BIG;
init_cmp_num_class_p = 0;
@@ -1008,10 +1010,13 @@ static ETERM *erl_decode_it(unsigned char **ext)
return ep;
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
ERL_TYPE(ep) = ERL_FLOAT;
- if (sscanf((char *) *ext, "%lf", &ff) != 1)
- goto failure;
- *ext += 31;
+ cp = (char *) *ext;
+ i = -1;
+ if (ei_decode_double(cp, &i, &ff) == -1)
+ goto failure;
+ *ext += i;
ep->uval.fval.f = ff;
return ep;
@@ -1176,6 +1181,7 @@ unsigned char erl_ext_type(unsigned char *ext)
case ERL_LARGE_TUPLE_EXT:
return ERL_TUPLE;
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
return ERL_FLOAT;
case ERL_BINARY_EXT:
return ERL_BINARY;
@@ -1218,6 +1224,7 @@ int erl_ext_size(unsigned char *t)
case ERL_BINARY_EXT:
case ERL_STRING_EXT:
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
case ERL_SMALL_BIG_EXT:
case ERL_LARGE_BIG_EXT:
return 0;
@@ -1332,6 +1339,9 @@ static int jump(unsigned char **ext)
case ERL_FLOAT_EXT:
*ext += 31;
break;
+ case NEW_FLOAT_EXT:
+ *ext += 8;
+ break;
case ERL_BINARY_EXT:
i = (**ext << 24) | ((*ext)[1] << 16) |((*ext)[2] << 8) | (*ext)[3];
*ext += 4+i;
@@ -1696,12 +1706,15 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
}
return 0;
case ERL_FLOAT_EXT:
- if (sscanf((char *) *e1, "%lf", &ff1) != 1)
- return -1;
- *e1 += 31;
- if (sscanf((char *) *e2, "%lf", &ff2) != 1)
- return -1;
- *e2 += 31;
+ case NEW_FLOAT_EXT:
+ i = -1;
+ if (ei_decode_double((char *) *e1, &i, &ff1) != 0)
+ return -1;
+ *e1 += i;
+ j = -1;
+ if (ei_decode_double((char *) *e2, &j, &ff2) != 0)
+ return -1;
+ *e2 += j;
return cmp_floats(ff1,ff2);
case ERL_BINARY_EXT:
@@ -25,9 +25,9 @@
#include "ei_decode_term.h"
#include "putget.h"
-/* Returns 1 if term is decoded, 0 if term is OK, but not decoded here
- and -1 if something is wrong.
- ONLY changes index if term is decoded (return value 1)! */
+/* Returns 0 on successful encoding, -1 on error, and 1 if the term seems
+ alright, but does not fit in the term structure. If it returns 0, the
+ index will be incremented, and the term contains the decoded term. */
int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
{
@@ -46,11 +46,8 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
term->value.i_val = get32be(s);
break;
case ERL_FLOAT_EXT:
- if (s[30]) return -1;
- if (sscanf(s, "%lf", &f) != 1) return -1;
- s += 31;
- term->value.d_val = f;
- break;
+ case NEW_FLOAT_EXT:
+ return ei_decode_double(buf, index, &term->value.d_val);
case ERL_ATOM_EXT:
len = get16be(s);
memcpy(term->value.atom_name, s, len);
@@ -272,6 +272,7 @@ static int print_term(FILE* fp, ei_x_buff* x,
break;
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
if (ei_decode_double(buf, index, &d) < 0) goto err;
ch_written += xprintf(fp, x, "%f", d);
break;
@@ -54,6 +54,18 @@
(s) += 4; \
} while (0)
+#define put64be(s,n) do { \
+ (s)[0] = ((n) >> 56) & 0xff; \
+ (s)[1] = ((n) >> 48) & 0xff; \
+ (s)[2] = ((n) >> 40) & 0xff; \
+ (s)[3] = ((n) >> 32) & 0xff; \
+ (s)[4] = ((n) >> 24) & 0xff; \
+ (s)[5] = ((n) >> 16) & 0xff; \
+ (s)[6] = ((n) >> 8) & 0xff; \
+ (s)[7] = (n) & 0xff; \
+ (s) += 8; \
+} while (0)
+
#define get8(s) \
((s) += 1, \
((unsigned char *)(s))[-1] & 0xff)
@@ -82,4 +94,20 @@
(((unsigned char *)(s))[-2] << 8) | \
((unsigned char *)(s))[-1]))
+#define get64be(s) \
+ ((s) += 8, \
+ (((EI_ULONGLONG)((unsigned char *)(s))[-8] << 56) | \
+ ((EI_ULONGLONG)((unsigned char *)(s))[-7] << 48) | \
+ ((EI_ULONGLONG)((unsigned char *)(s))[-6] << 40) | \
+ ((EI_ULONGLONG)((unsigned char *)(s))[-5] << 32) | \
+ ((EI_ULONGLONG)((unsigned char *)(s))[-4] << 24) | \
+ ((EI_ULONGLONG)((unsigned char *)(s))[-3] << 16) | \
+ ((EI_ULONGLONG)((unsigned char *)(s))[-2] << 8) | \
+ (EI_ULONGLONG)((unsigned char *)(s))[-1]))
+
+typedef union float_ext {
+ double d;
+ EI_ULONGLONG val;
+} FloatExt;
+
#endif /* _PUTGET_H */
@@ -400,6 +400,7 @@ static void show_term(const char *termbuf, int *index, FILE *stream)
break;
case ERL_FLOAT_EXT:
+ case NEW_FLOAT_EXT:
ei_decode_double(termbuf,index,&fnum);
fprintf(stream,"%f",fnum);
break;
@@ -181,22 +181,9 @@ test_ei_decode_misc(suite) -> [];
test_ei_decode_misc(Config) when is_list(Config) ->
?line P = runner:start(?test_ei_decode_misc),
-% ?line <<131>> = get_binaries(P),
-
-% ?line {term,F} = get_term(P),
-% ?line match_float(F, 0.0),
-% ?line {term,F} = get_term(P),
-% ?line match_float(F, 0.0),
-
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, -1.0),
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, -1.0),
-
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, 1.0),
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, 1.0),
+ ?line send_term_as_binary(P,0.0),
+ ?line send_term_as_binary(P,-1.0),
+ ?line send_term_as_binary(P,1.0),
?line send_term_as_binary(P,false),
?line send_term_as_binary(P,true),
@@ -515,11 +515,10 @@ TESTCASE(test_ei_decode_misc)
/*
EI_DECODE_0(decode_version);
*/
-/*
- EI_DECODE_2(decode_double, 0.0);
- EI_DECODE_2(decode_double, -1.0);
- EI_DECODE_2(decode_double, 1.0);
-*/
+ EI_DECODE_2(decode_double, 32, double, 0.0);
+ EI_DECODE_2(decode_double, 32, double, -1.0);
+ EI_DECODE_2(decode_double, 32, double, 1.0);
+
EI_DECODE_2(decode_boolean, 8, int, 0);
EI_DECODE_2(decode_boolean, 7, int, 1);
@@ -181,20 +181,14 @@ test_ei_encode_misc(Config) when is_list(Config) ->
?line <<131>> = get_binaries(P),
-% ?line {term,F} = get_term(P),
-% ?line match_float(F, 0.0),
-% ?line {term,F} = get_term(P),
-% ?line match_float(F, 0.0),
+ ?line {<<70,_:8/binary>>,F0} = get_buf_and_term(P),
+ ?line true = match_float(F0, 0.0),
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, -1.0),
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, -1.0),
+ ?line {<<70,_:8/binary>>,Fn1} = get_buf_and_term(P),
+ ?line true = match_float(Fn1, -1.0),
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, 1.0),
-% ?line {term,F} = get_term(P),
-% ?line true = match_float(F, 1.0),
+ ?line {<<70,_:8/binary>>,Fp1} = get_buf_and_term(P),
+ ?line true = match_float(Fp1, 1.0),
?line {<<100,0,5,"false">>,false} = get_buf_and_term(P),
?line {<<100,0,4,"true">> ,true} = get_buf_and_term(P),
@@ -310,6 +304,8 @@ get_term(P) ->
%%
+match_float(F, Match) when is_float(F), is_float(Match), F == Match ->
+ true;
match_float(F, Match) when is_float(F), F > Match*0.99, F < Match*1.01 ->
true.
@@ -350,13 +350,13 @@ TESTCASE(test_ei_encode_char)
TESTCASE(test_ei_encode_misc)
{
EI_ENCODE_0(encode_version);
-/*
+
EI_ENCODE_1(encode_double, 0.0);
EI_ENCODE_1(encode_double, -1.0);
EI_ENCODE_1(encode_double, 1.0);
-*/
+
EI_ENCODE_1(encode_boolean, 0) /* Only case it should be false */;
EI_ENCODE_1(encode_boolean, 1);
@@ -1637,7 +1637,7 @@ static Boolean decode_params(db_state *state, byte *buffer, int *index, param_ar
break;
case SQL_C_DOUBLE:
- if((erl_type != ERL_FLOAT_EXT)) {
+ if((erl_type != ERL_FLOAT_EXT) && (erl_type != NEW_FLOAT_EXT)) {
return FALSE;
}
ei_decode_double(buffer, index, &(param->values.floating[j]));

0 comments on commit f9eeacc

Please sign in to comment.