-
Notifications
You must be signed in to change notification settings - Fork 5
/
processing.go
88 lines (77 loc) · 2.48 KB
/
processing.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
package processing
import (
"bufio"
"bytes"
"github.com/alessiosavi/GoGPUtils/helper"
"io"
"io/ioutil"
"log"
)
type LineTerminatorType string
const (
LF LineTerminatorType = "\n" // 0a -- 10
CR LineTerminatorType = "\r" // 0d -- 13
CRLF LineTerminatorType = "\r\n" // 0d,0a -- 13,10
LFCR LineTerminatorType = "\n\r" // 0a,0d -- 10,13
RS LineTerminatorType = "\036" // 1e -- 30
ND LineTerminatorType = `unable to detect line terminator`
)
func DetectLineTerminator(reader io.Reader) (LineTerminatorType, error) {
// Read 1mb of file
log.SetFlags(log.Llongfile | log.LstdFlags)
buff := make([]byte, 1024*1000)
var counts map[LineTerminatorType]int = make(map[LineTerminatorType]int)
for {
if _, err := reader.Read(buff); err != nil {
if err != io.EOF {
return ND, err
} else {
break
}
}
counts[CRLF] = bytes.Count(buff, []byte("\r\n"))
counts[LFCR] = bytes.Count(buff, []byte("\n\r"))
counts[CR] = bytes.Count(buff, []byte("\r"))
counts[LF] = bytes.Count(buff, []byte("\n"))
counts[RS] = bytes.Count(buff, []byte("\036"))
}
counts[CR] -= counts[CRLF] + counts[LFCR]
counts[LF] -= counts[CRLF] + counts[LFCR]
maxV := 0
var maxKey LineTerminatorType = ND
for k, v := range counts {
if v > maxV {
maxV = v
maxKey = k
}
}
log.Println(helper.MarshalIndent(counts))
return maxKey, nil
}
// ReplaceLineTerminator is delegated to find the line terminator of the given byte array and replace them without the one provided in input
func ReplaceLineTerminator(data, newLineTerminator []byte) ([]byte, error) {
terminator, err := DetectLineTerminator(bytes.NewBuffer(data))
if err != nil {
return nil, err
}
newData := bytes.ReplaceAll(data, []byte(terminator), newLineTerminator)
newData = bytes.TrimSpace(newData)
return bytes.Trim(newData, string(newLineTerminator)), nil
}
// ReplaceLineTerminator is delegated to find the line terminator of the given byte array and replace them without the one provided in input
func ReplaceLineTerminatorBytesReader(data *bytes.Reader, newLineTerminator []byte) ([]byte, error) {
terminator, err := DetectLineTerminator(bufio.NewReader(data))
if err != nil {
return nil, err
}
// Read all the file
newData, err := ioutil.ReadAll(data)
if err != nil {
return nil, err
}
// Reset to the start of the file
data.Seek(0, io.SeekStart)
newData = bytes.ReplaceAll(newData, []byte(terminator), newLineTerminator)
newData = bytes.TrimSpace(newData)
return bytes.Trim(newData, string(newLineTerminator)), nil
}