escape control characters in JSON; fixes #165

pkg/writer/writer.go

@@ -126,7 +126,6 @@ func (c *csvWriter) File(name string, sz int64, mod string, checksum []byte, err
 		c.w.Write(r)
 	}
 	c.recs = c.recs[:1]
-	return
 }
 
 func (c *csvWriter) Tail() { c.w.Flush() }
@@ -226,8 +225,37 @@ type jsonWriter struct {
 
 func JSON(w io.Writer) Writer {
 	return &jsonWriter{
-		replacer: strings.NewReplacer(`"`, `\"`, `\\`, `\\`, `\`, `\\`),
-		w:        bufio.NewWriter(w),
+		replacer: strings.NewReplacer(
+			`\`, `\\`,
+			`"`, `\"`,
+			"\u0000", `\u0000`,
+			"\u0001", `\u0001`,
+			"\u0002", `\u0002`,
+			"\u0003", `\u0003`,
+			"\u0004", `\u0004`,
+			"\u0005", `\u0005`,
+			"\u0006", `\u0006`,
+			"\u0007", `\u0007`,
+			"\u0008", `\u0008`,
+			"\u0009", `\u0009`,
+			"\u000A", `\u000A`,
+			"\u000B", `\u000B`,
+			"\u000C", `\u000C`,
+			"\u000D", `\u000D`,
+			"\u000E", `\u000E`,
+			"\u000F", `\u000F`,
+			"\u0010", `\u0010`,
+			"\u0011", `\u0011`,
+			"\u0012", `\u0012`,
+			"\u0013", `\u0013`,
+			"\u0014", `\u0014`,
+			"\u0015", `\u0015`,
+			"\u0016", `\u0016`,
+			"\u0017", `\u0017`,
+			"\u0018", `\u0018`,
+			"\u0019", `\u0019`,
+		),
+		w: bufio.NewWriter(w),
 	}
 }
 
@@ -299,7 +327,6 @@ func (j *jsonWriter) File(name string, sz int64, mod string, checksum []byte, er
 	}
 	j.w.WriteString("]}")
 	j.subs = true
-	return
 }
 
 func (j *jsonWriter) Tail() {
@@ -389,7 +416,6 @@ func (d *droidWriter) File(p string, sz int64, mod string, checksum []byte, err
 		d.rec[3] = clearArchivePath(d.rec[2], d.rec[3])
 		d.w.Write(d.rec)
 	}
-	return
 }
 
 func (d *droidWriter) Tail() { d.w.Flush() }

pkg/writer/writer_test.go

@@ -16,6 +16,9 @@ package writer
 
 import (
 	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -55,6 +58,51 @@ func makeFields() []string {
 		"warning"}
 }
 
+var controlCharacters = []string{"\u0000", "\u0001", "\u0002", "\u0003",
+	"\u0004", "\u0005", "\u0006", "\u0007", "\u0008", "\u0009", "\u000A",
+	"\u000B", "\u000C", "\u000D", "\u000E", "\u000F", "\u0010", "\u0011",
+	"\u0012", "\u0013", "\u0014", "\u0015", "\u0016", "\u0017", "\u0018",
+	"\u0019",
+}
+var nonControlCharacters = []string{"\u0020", "\u1F5A4", "\u265B", "\u1F0A1",
+	"\u262F",
+}
+
+// TestJSONControlCharacters tests control characters that are valid but
+// need special treatment from the writer and makes sure that they
+// create invalid JSON.
+func TestControlCharacters(t *testing.T) {
+	buf := &bytes.Buffer{}
+	js := JSON(buf)
+	js.Head("", time.Time{}, time.Time{}, [3]int{}, [][2]string{{"pronom", ""}}, [][]string{makeFields()}, "")
+	// Loop through the control characters to make sure the JSON output
+	// is valid.
+	for _, val := range controlCharacters {
+		js.File(fmt.Sprintf("path/%sto/file", val), 1, "2015-05-24T16:59:13+10:00", nil, testErr{}, []core.Identification{testID{}})
+	}
+	js.Tail()
+	if !json.Valid(buf.Bytes()) {
+		t.Errorf("Invalid JSON:\n%s", buf.String())
+	}
+}
+
+// TestNonControlCharacters tests valid characters and simply makes sure
+// that the JSON output is correct.
+func TestNonControlCharacters(t *testing.T) {
+	buf := &bytes.Buffer{}
+	js := JSON(buf)
+	js.Head("", time.Time{}, time.Time{}, [3]int{}, [][2]string{{"pronom", ""}}, [][]string{makeFields()}, "")
+	// Loop through the non control characters to make sure the JSON output
+	// is valid.
+	for _, val := range nonControlCharacters {
+		js.File(fmt.Sprintf("path/%sto/file", val), 1, "2015-05-24T16:59:13+10:00", nil, testErr{}, []core.Identification{testID{}})
+	}
+	js.Tail()
+	if !json.Valid(buf.Bytes()) {
+		t.Errorf("Invalid JSON:\n%s", buf.String())
+	}
+}
+
 func TestYAMLHeader(t *testing.T) {
 	expect := "  - ns      : %v\n    id      : %v\n    format  : %v\n    version : %v\n    mime    : %v\n    basis   : %v\n    warning : %v\n"
 	ret := header(makeFields())