OS-5047 nvlist_print_json assertion failure

Reviewed by: Joshua M. Clulow <jmc@joyent.com>

usr/src/lib/libnvpair/nvpair_json.c

@@ -38,7 +38,7 @@ nvlist_rasnprintf(char **bufp, size_t *blen, off_t *boff, char *input, ...)
 {
 	int ret;
 	va_list ap;
-	size_t size, asize;
+	size_t size;
 	char *b;
 
 	if (*bufp == NULL) {
@@ -56,13 +56,12 @@ nvlist_rasnprintf(char **bufp, size_t *blen, off_t *boff, char *input, ...)
 	/* E_SEC_PRINTF_VAR_FMT */
 	ret = vsnprintf(*bufp + *boff, size, input, ap);
 	va_end(ap);
-	if (ret < 0) {
-		va_end(ap);
+	if (ret < 0)
 		return (-1);
-	}
+
 	if (ret >= size) {
-		asize = *blen;
-		while (ret >= asize)
+		size_t asize = *blen;
+		while (ret + *boff >= asize)
 			asize += 1024;
 		if ((b = realloc(*bufp, asize)) == NULL)
 			return (-1);

usr/src/test/util-tests/tests/libnvpair_json/json_08_large_data.ksh

@@ -0,0 +1,37 @@
+#!/bin/ksh
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2015, Joyent, Inc.
+#
+
+DIR=$(dirname $(whence $0))
+. ${DIR}/json_common
+
+BASELINE="$(cat <<EOF
+{\
+"o":[\
+$(for i in {1..50}; do
+    printf '"%s",' 'some_very_big_string_that_takes_up_space'
+done)\
+"end"\
+]\
+}
+EOF)"
+
+OUTPUT="$(${DIR}/../../bin/print_json <<-EOF
+add_string_array "o" $(for i in {1..50}; do
+    printf '"%s" ' 'some_very_big_string_that_takes_up_space'
+done)"end";
+EOF)"
+
+complete