text_utils.go

package main

import (
	"bytes"
	"io"
	"io/ioutil"
	"strings"

	//"github.com/saintfish/chardet" // not work, really stupid...
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/htmlindex"
	"golang.org/x/text/transform"
)

// mime type struct
type mimeType struct {
	Type    string
	subType string
	scope   string
}

// parse mime type
func parseMimeType(contentTypeStr string) mimeType {
	var idx = strings.Index(contentTypeStr, "/")
	if idx == -1 {
		// should not happen
		return mimeType{contentTypeStr, "", ""}
	}
	var scope = ""
	var subType = contentTypeStr[idx+1:]
	if strings.HasPrefix(subType, "x-") {
		subType = subType[2:]
		scope = "x"
	} else if strings.HasPrefix(subType, "vnd.") {
		subType = subType[4:]
		scope = "vnd"
	}
	var i = strings.Index(subType, ".")
	if i > 0 {
		subType = subType[:i]
	}
	return mimeType{contentTypeStr[:idx], subType, scope}
}

//TODO: multipart/form-data

var textTypes = map[string]bool{"text": true}
var textSubTypes = map[string]bool{
	"html":                true,
	"xml":                 true,
	"json":                true,
	"www-form-urlencoded": true,
	"javascript":          true,
	"postscript":          true,
	"atomcat+xml":         true,
	"atomsvc+xml":         true,
	"atom+xml":            true,
	"xml-dtd":             true,
	"ecmascript":          true,
	"java-jnlp-file":      true,
	"latex":               true,
	"mpegurl":             true,
	"rdf+xml":             true,
	"rtf":                 true,
	"rss+xml":             true,
	"svg+xml":             true,
	"uri-list":            true,
	"wsdl+xml":            true,
	"xhtml+xml":           true,
	"xslt+xml":            true,
	"ns-proxy-autoconfig": true,
	"javascript-config":   true,
}

// if is text type mime
func (ct mimeType) isTextContent() bool {
	return textTypes[ct.Type] || textSubTypes[ct.subType]
}

var binaryTypes = map[string]bool{"image": true, "audio": true, "video": true}
var binarySubtypes = map[string]bool{
	"7z-compressed":                 true,
	"abiword":                       true,
	"ace-compressed":                true,
	"shockwave-flash":               true,
	"pdf":                           true,
	"director":                      true,
	"bzip":                          true,
	"bzip2":                         true,
	"debian-package":                true,
	"epub+zip":                      true,
	"font-ghostscript":              true,
	"font-bdf":                      true,
	"java-archive":                  true,
	"java-vm":                       true,
	"java-serialized-object":        true,
	"msaccess":                      true,
	"msdownload":                    true,
	"ms-application":                true,
	"ms-fontobject":                 true,
	"ms-excel":                      true,
	"openxmlformats-officedocument": true,
	"msbinder":                      true,
	"ms-officetheme":                true,
	"onenote":                       true,
	"ms-powerpoint":                 true,
	"ms-project":                    true,
	"mspublisher":                   true,
	"msschedule":                    true,
	"silverlight-app":               true,
	"visio":                         true,
	"ms-wmd":                        true,
	"ms-htmlhelp":                   true,
	"msword":                        true,
	"ms-works":                      true,
	"oda":                           true,
	"ogg":                           true,
	"oasis":                         true,
	"sun":                           true,
	"font-otf":                      true,
	"x-font-ttf":                    true,
	"unity":                         true,
	"zip":                           true,
	"x509-ca-cert":                  true,
	"octet-stream":                  true,
	"png":                           true,
	"ppt":                           true,
	"xls":                           true,
}

// if is binary type mime
func (ct mimeType) isBinaryContent() bool {
	return binaryTypes[ct.Type] || binarySubtypes[ct.subType]
}

// read reader content to string, using charset specified
func readToStringWithCharset(reader io.Reader, charset string) (string, error) {
	charset = strings.ToUpper(charset)
	var data []byte
	var err error
	if charset == "UTF-8" || charset == "UTF8" {
		data, err = ioutil.ReadAll(reader)
	} else {
		if charset == "GBK" || charset == "GB2312" {
			charset = "GB18030"
		}
		var encoder encoding.Encoding
		encoder, err = htmlindex.Get(charset)
		if err != nil {
			return "", err
		}
		data, err = ioutil.ReadAll(transform.NewReader(reader, encoder.NewDecoder()))
	}
	if err != nil {
		return "", err
	}
	return string(data), err
}

func readToBytesWithCharset(reader io.Reader, charset string) ([]byte, error) {
	charset = strings.ToUpper(charset)
	var data []byte
	var err error
	if charset == "UTF-8" || charset == "UTF8" {
		data, err = ioutil.ReadAll(reader)
	} else {
		if charset == "GBK" || charset == "GB2312" {
			charset = "GB18030"
		}
		var encoder encoding.Encoding
		encoder, err = htmlindex.Get(charset)
		if err != nil {
			return []byte(""), err
		}
		data, err = ioutil.ReadAll(transform.NewReader(reader, encoder.NewDecoder()))
	}
	if err != nil {
		return []byte(""), err
	}
	return data, err
}

// convert byte array to string, using charset specified
func byteToStringWithCharset(data []byte, charset string) (string, error) {
	charset = strings.ToUpper(charset)
	if charset == "UTF-8" || charset == "UTF8" {
		return string(data), nil
	}
	var reader = bytes.NewBuffer(data)
	return readToStringWithCharset(reader, charset)
}

// parse content type to mimeType and charset
func parseContentType(contentType string) (string, string) {
	var mimeTypeStr, charset string
	idx := strings.Index(contentType, ";")
	if idx < 0 {
		mimeTypeStr = strings.TrimSpace(contentType)
		charset = ""
	} else {
		mimeTypeStr = strings.TrimSpace(contentType[:idx])
		charsetSeg := strings.TrimSpace(contentType[idx+1:])
		eidx := strings.Index(charsetSeg, "=")
		if eidx < 0 {
			charset = ""
		} else {
			charset = strings.TrimSpace(charsetSeg[eidx+1:])
		}
	}
	return mimeTypeStr, charset
}

// if sting 'looks like' a json string
func likeJSON(value string) bool {
	if len(value) < 2 {
		return false
	}
	value = strings.TrimSpace(value)
	if value[0] == '[' && value[len(value)-1] == ']' || value[0] == '{' && value[len(value)-1] == '}' {
		return true
	}
	return false
}

// if str wildcard match pattern
func wildcardMatch(str string, pattern string) bool {
	var n = len(pattern)
	var i = 0
	var j = 0
	var asterick = -1
	var match = 0
	for i < len(str) {
		if j < n && pattern[j] == '*' {
			match = i
			asterick = j
			j++
		} else if j < n && (str[i] == pattern[j] || pattern[j] == '?') {
			i++
			j++
		} else if asterick >= 0 {
			match++
			i = match
			j = asterick + 1
		} else {
			return false
		}
	}
	for j < n && pattern[j] == '*' {
		j++
	}
	return j == n
}