strip invalid utf8 before writing to db

backends/rapidpro/backend_test.go

@@ -429,7 +429,7 @@ func (ts *BackendTestSuite) TestChanneLog() {
 	knChannel := ts.getChannel("KN", "dbc126ed-66bc-4e28-b67b-81dc3327c95d")
 
 	log := courier.NewChannelLog("Message Send Error", knChannel, courier.NilMsgID, "POST", "/null/value", 400,
-		"request with null \x00", "response with null \x00", time.Millisecond, nil)
+		"request with null \x00 content", "response with null \x00 content", time.Millisecond, nil)
 
 	err := writeChannelLog(ts.b, log)
 	ts.NoError(err)

backends/rapidpro/log.go

@@ -2,11 +2,11 @@ package rapidpro
 
 import (
 	"fmt"
-	"strings"
 
 	"time"
 
 	"github.com/nyaruka/courier"
+	"github.com/nyaruka/courier/utils"
 )
 
 const insertLogSQL = `
@@ -28,8 +28,8 @@ func writeChannelLog(b *backend, log *courier.ChannelLog) error {
 	}
 
 	// strip null chars from request and response, postgres doesn't like that
-	log.Request = strings.Trim(log.Request, "\x00")
-	log.Response = strings.Trim(log.Response, "\x00")
+	log.Request = utils.CleanString(log.Request)
+	log.Response = utils.CleanString(log.Response)
 
 	_, err := b.db.Exec(insertLogSQL, dbChan.ID(), log.MsgID, log.Description, log.Error != "", log.Method, log.URL,
 		log.Request, log.Response, log.StatusCode, log.CreatedOn, log.Elapsed/time.Millisecond)

sender.go

@@ -88,7 +88,7 @@ func (f *Foreman) Assign() {
 
 				// add our sender back to our queue and sleep a bit
 				if !lastSleep {
-					log.Info("sleeping, no messages")
+					log.Debug("sleeping, no messages")
 					lastSleep = true
 				}
 				f.availableSenders <- sender

utils/misc.go

@@ -63,5 +63,22 @@ var invalidChars = regexp.MustCompile("([\u0000-\u0008]|[\u000B-\u000C]|[\u000E-
 
 // CleanString removes any control characters from the passed in string
 func CleanString(s string) string {
-	return invalidChars.ReplaceAllString(s, "")
+	cleaned := invalidChars.ReplaceAllString(s, "")
+
+	// check whether this is valid UTF8
+	if !utf8.ValidString(cleaned) {
+		v := make([]rune, 0, len(cleaned))
+		for i, r := range s {
+			if r == utf8.RuneError {
+				_, size := utf8.DecodeRuneInString(s[i:])
+				if size == 1 {
+					continue
+				}
+			}
+			v = append(v, r)
+		}
+		cleaned = string(v)
+	}
+
+	return cleaned
 }

utils/misc_test.go

@@ -25,4 +25,6 @@ func TestStringArrayContains(t *testing.T) {
 func TestCleanString(t *testing.T) {
 	assert.Equal(t, "\x41hello", CleanString("\x02\x41hello"))
 	assert.Equal(t, "😅 happy!", CleanString("😅 happy!"))
+	assert.Equal(t, "Hello  There", CleanString("Hello \x00 There"))
+	assert.Equal(t, "Hello z There", CleanString("Hello \xc5z There"))
 }