-
Notifications
You must be signed in to change notification settings - Fork 70
/
concatenator.go
124 lines (109 loc) · 3.42 KB
/
concatenator.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
package components
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/scipipe/scipipe"
)
// Concatenator is a process that concatenates the content of multiple files
// received in the in-port In, into one file returned on its out-port, Out.
// You can optionally specify a tag name to GroupByTag, which will make files
// go into separate output files if they have different values for that tag.
// These output files will have the tag name appended to the base file name.
type Concatenator struct {
scipipe.BaseProcess
OutPath string
GroupByTag string
}
// NewConcatenator returns a new, initialized Concatenator process
func NewConcatenator(wf *scipipe.Workflow, name string, outPath string) *Concatenator {
p := &Concatenator{
BaseProcess: scipipe.NewBaseProcess(wf, name),
OutPath: outPath,
}
p.InitInPort(p, "in")
p.InitOutPort(p, "out")
wf.AddProc(p)
return p
}
// In returns the (only) in-port for this process
func (p *Concatenator) In() *scipipe.InPort { return p.InPort("in") }
// Out returns the (only) out-port for this process
func (p *Concatenator) Out() *scipipe.OutPort { return p.OutPort("out") }
// Run runs the Concatenator process
func (p *Concatenator) Run() {
defer p.CloseAllOutPorts()
outIP, err := scipipe.NewFileIP(p.OutPath)
if err != nil {
p.Fail(err)
}
oipDir := filepath.Dir(outIP.Path())
err = os.MkdirAll(oipDir, 0777)
if err != nil {
p.Failf("Could not create directory: (%s) for out-IP (%s):\n%s", oipDir, outIP.Path(), err)
}
outFh, err := os.Create(outIP.Path())
if err != nil {
p.Failf("Could not open path for writing: %s\n", outIP.Path())
}
outIPsByTag := make(map[string]*scipipe.FileIP)
outFhsByTag := make(map[string]*os.File)
for inIP := range p.In().Chan {
tagVal := inIP.Tag(p.GroupByTag)
if tagVal != "" {
if _, ok := outIPsByTag[tagVal]; !ok {
outIPForTagPath := fmt.Sprintf("%s.%s_%s", p.OutPath, p.GroupByTag, tagVal)
outIPForTag, err := scipipe.NewFileIP(outIPForTagPath)
if err != nil {
p.Failf("Could not create FileIP with path: %s\nOriginal error: %v", outIPForTagPath, err)
}
outIPForTag.AddTag(p.GroupByTag, tagVal)
outIPsByTag[tagVal] = outIPForTag
outFh, err := os.Create(outIPForTag.Path())
if err != nil {
p.Failf("Could not create path: %s\nOriginal error: %v", outIPForTag.Path(), err)
}
outFhsByTag[tagVal] = outFh
}
dat, err := ioutil.ReadFile(inIP.Path())
if err != nil {
p.Failf("Could not read file: %s\n", inIP.Path())
}
outFhsByTag[tagVal].Write(append(dat))
if err != nil {
p.Failf("Could not write to file: %s\n", outIPsByTag[tagVal].Path())
}
outFhsByTag[tagVal].Write(append([]byte("\n")))
if err != nil {
p.Failf("Could not write to file: %s\n", outIPsByTag[tagVal].Path())
}
} else {
dat, err := ioutil.ReadFile(inIP.Path())
if err != nil {
p.Failf("Could not read file: %s\n", inIP.Path())
}
_, err = outFh.Write(append(dat))
if err != nil {
p.Failf("Could not write to file: %s\n", outIP.Path())
}
_, err = outFh.Write(append([]byte("\n")))
if err != nil {
p.Failf("Could not write to file: %s\n", outIP.Path())
}
}
}
// Close file handles
err = outFh.Close()
if err != nil {
p.Failf("Could not close file handle: %s\n", outIP.Path())
}
for _, taggedFh := range outFhsByTag {
taggedFh.Close()
}
// Send IPs
p.Out().Send(outIP)
for _, taggedIP := range outIPsByTag {
p.Out().Send(taggedIP)
}
}