11280 custr in belt but not braces, bungles vsnprintf() buffer length

Reviewed by: Jason King <jason.king@joyent.com>
Reviewed by: Andy Fiddaman <omnios@citrus-it.co.uk>
Approved by: Richard Lowe <richlowe@richlowe.net>

usr/src/lib/libcustr/common/custr.c

@@ -14,7 +14,7 @@
  */
 
 /*
- * Copyright 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #include <stdlib.h>
@@ -78,8 +78,9 @@ custr_append_vprintf(custr_t *cus, const char *fmt, va_list ap)
 	int len = vsnprintf(NULL, 0, fmt, ap);
 	size_t chunksz = STRING_CHUNK_SIZE;
 
-	if (len == -1)
+	if (len < 0) {
 		return (len);
+	}
 
 	while (chunksz < len) {
 		chunksz *= 2;
@@ -120,10 +121,10 @@ custr_append_vprintf(custr_t *cus, const char *fmt, va_list ap)
 	/*
 	 * Append new string to existing string:
 	 */
-	len = vsnprintf(cus->cus_data + cus->cus_strlen,
-	    (uintptr_t)cus->cus_data - (uintptr_t)cus->cus_strlen, fmt, ap);
-	if (len == -1)
+	if ((len = vsnprintf(cus->cus_data + cus->cus_strlen,
+	    cus->cus_datalen - cus->cus_strlen, fmt, ap)) < 0) {
 		return (len);
+	}
 	cus->cus_strlen += len;
 
 	return (0);