Add warning for BIDI characters in page renders and in diffs

modules/charset/charset.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 
 	"code.gitea.io/gitea/modules/log"
@@ -18,11 +19,215 @@ import (
 	"github.com/gogs/chardet"
 	"golang.org/x/net/html/charset"
 	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/bidi"
 )
 
 // UTF8BOM is the utf-8 byte-order marker
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
+// EscapeStatus represents the findings of the unicode escaper
+type EscapeStatus struct {
+	Escaped      bool
+	HasError     bool
+	HasBadRunes  bool
+	HasControls  bool
+	HasSpaces    bool
+	HasMarks     bool
+	HasBIDI      bool
+	BadBIDI      bool
+	HasRTLScript bool
+	HasLTRScript bool
+}
+
+// Or combines two EscapeStatus structs into one representing the conjunction of the two
+func (status EscapeStatus) Or(other EscapeStatus) EscapeStatus {
+	status.Escaped = status.Escaped || other.Escaped
+	status.HasError = status.HasError || other.HasError
+	status.HasBadRunes = status.HasBadRunes || other.HasBadRunes
+	status.HasControls = status.HasControls || other.HasControls
+	status.HasSpaces = status.HasSpaces || other.HasSpaces
+	status.HasMarks = status.HasMarks || other.HasMarks
+	status.HasBIDI = status.HasBIDI || other.HasBIDI
+	status.BadBIDI = status.BadBIDI || other.BadBIDI
+	status.HasRTLScript = status.HasRTLScript || other.HasRTLScript
+	status.HasLTRScript = status.HasLTRScript || other.HasLTRScript
+	return status
+}
+
+// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
+func EscapeControlString(text string) (EscapeStatus, string) {
+	sb := &strings.Builder{}
+	escaped, _ := EscapeControlReader(strings.NewReader(text), sb)
+	return escaped, sb.String()
+}
+
+// EscapeControlBytes escapes the unicode control sequences  a provided []byte and returns the findings as an EscapeStatus and the escaped []byte
+func EscapeControlBytes(text []byte) (EscapeStatus, []byte) {
+	buf := &bytes.Buffer{}
+	escaped, _ := EscapeControlReader(bytes.NewReader(text), buf)
+	return escaped, buf.Bytes()
+}
+
+// EscapeControlReader escapes the unicode control sequences  a provided Reader writing the escaped output to the output and returns the findings as an EscapeStatus and an error
+func EscapeControlReader(text io.Reader, output io.Writer) (escaped EscapeStatus, err error) {
+	buf := make([]byte, 4096)
+	readStart := 0
+	var n int
+	var writePos int
+
+	lineHasBIDI := false
+	lineHasRTLScript := false
+	lineHasLTRScript := false
+
+readingloop:
+	for err == nil {
+		n, err = text.Read(buf[readStart:])
+		bs := buf[:n+readStart]
+		i := 0
+
+		for i < len(bs) {
+			r, size := utf8.DecodeRune(bs[i:])
+			// Now handle the codepoints
+			switch {
+			case r == utf8.RuneError:
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+					writePos = i
+				}
+				// runes can be at most 4 bytes - so...
+				if len(bs)-i <= 3 {
+					// if not request more data
+					copy(buf, bs[i:])
+					readStart = n - i
+					writePos = 0
+					continue readingloop
+				}
+				// this is a real broken rune
+				escaped.HasBadRunes = true
+				escaped.Escaped = true
+				if _, err = fmt.Fprintf(output, `<span class="broken-code-point">&lt;%X&gt;</span>`, bs[i:i+size]); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos += size
+			case r == '\n':
+				if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript {
+					escaped.BadBIDI = true
+				}
+				lineHasBIDI = false
+				lineHasRTLScript = false
+				lineHasLTRScript = false
+
+			case r == '\r' || r == '\t' || r == ' ':
+				// These are acceptable control characters and space characters
+			case unicode.IsSpace(r):
+				escaped.HasSpaces = true
+				escaped.Escaped = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				if _, err = fmt.Fprintf(output, `<span class="escaped-code-point" escaped="[U+%04X]"><span class="char">%c</span></span>`, r, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			case unicode.Is(unicode.Bidi_Control, r):
+				escaped.Escaped = true
+				escaped.HasBIDI = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				lineHasBIDI = true
+				if _, err = fmt.Fprintf(output, `<span class="escaped-code-point" escaped="[U+%04X]"><span class="char">%c</span></span>`, r, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			case unicode.Is(unicode.C, r):
+				escaped.Escaped = true
+				escaped.HasControls = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				if _, err = fmt.Fprintf(output, `<span class="escaped-code-point" escaped="[U+%04X]"><span class="char">%c</span></span>`, r, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			case unicode.Is(unicode.M, r):
+				escaped.Escaped = true
+				escaped.HasMarks = true
+				if writePos < i {
+					if _, err = output.Write(bs[writePos:i]); err != nil {
+						escaped.HasError = true
+						return
+					}
+				}
+				if _, err = fmt.Fprintf(output, `<span class="escaped-code-point" escaped="[U+%04X]"><span class="char">%c</span></span>`, r, r); err != nil {
+					escaped.HasError = true
+					return
+				}
+				writePos = i + size
+			default:
+				p, _ := bidi.Lookup(bs[i : i+size])
+				c := p.Class()
+				if c == bidi.R || c == bidi.AL {
+					lineHasRTLScript = true
+					escaped.HasRTLScript = true
+				} else if c == bidi.L {
+					lineHasLTRScript = true
+					escaped.HasLTRScript = true
+				}
+			}
+			i += size
+		}
+		if n > 0 {
+			// we read something...
+			// write everything unwritten
+			if writePos < i {
+				if _, err = output.Write(bs[writePos:i]); err != nil {
+					escaped.HasError = true
+					return
+				}
+			}
+
+			// reset the starting positions for the next read
+			readStart = 0
+			writePos = 0
+		}
+	}
+	if readStart > 0 {
+		// this means that there is an incomplete or broken rune at 0-readStart and we read nothing on the last go round
+		escaped.Escaped = true
+		escaped.HasBadRunes = true
+		if _, err = fmt.Fprintf(output, `<span class="broken-code-point">&lt;%X&gt;</span>`, buf[:readStart]); err != nil {
+			escaped.HasError = true
+			return
+		}
+	}
+	if err == io.EOF {
+		if lineHasBIDI && !lineHasRTLScript && lineHasLTRScript {
+			escaped.BadBIDI = true
+		}
+		err = nil
+		return
+	}
+	escaped.HasError = true
+	return
+}
+
 // ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
 func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
 	var buf = make([]byte, 2048)