ERR: fix err_data_size inconsistencies

In ERR_add_error_vdata(), the size of err_data had 1 added to it in
some spots, which could lead to buffer overflow.

In ERR_vset_error(), ERR_MAX_DATA_SIZE was used instead of buf_size in
the BIO_vsnprintf() call, which would lead to a buffer overflow if
such a large buffer couldn't be allocated.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from #9491)

crypto/err/err.c

@@ -795,18 +795,18 @@ void ERR_add_error_vdata(int num, va_list args)
         if (arg == NULL)
             arg = "<NULL>";
         len += strlen(arg);
-        if (len > size) {
+        if (len >= size) {
             char *p;
 
             size = len + 20;
-            p = OPENSSL_realloc(str, size + 1);
+            p = OPENSSL_realloc(str, size);
             if (p == NULL) {
                 OPENSSL_free(str);
                 return;
             }
             str = p;
         }
-        OPENSSL_strlcat(str, arg, (size_t)size + 1);
+        OPENSSL_strlcat(str, arg, (size_t)size);
     }
     if (!err_set_error_data_int(str, size, flags, 0))
         OPENSSL_free(str);

crypto/err/err_blocks.c

@@ -85,7 +85,7 @@ void ERR_vset_error(int lib, int reason, const char *fmt, va_list args)
         }
 
         if (buf != NULL) {
-            printed_len = BIO_vsnprintf(buf, ERR_MAX_DATA_SIZE, fmt, args);
+            printed_len = BIO_vsnprintf(buf, buf_size, fmt, args);
         }
         if (printed_len < 0)
             printed_len = 0;