/
concatenator.go
80 lines (69 loc) · 2.18 KB
/
concatenator.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package components
import (
"fmt"
"io/ioutil"
"os"
"github.com/scipipe/scipipe"
)
// Concatenator is a process that concatenates the content of multiple files
// received in the in-port In, into one file returned on its out-port, Out.
// You can optionally specify a tag name to GroupByTag, which will make files
// go into separate output files if they have different values for that tag.
// These output files will have the tag name appended to the base file name.
type Concatenator struct {
scipipe.BaseProcess
OutPath string
GroupByTag string
}
// NewConcatenator returns a new, initialized Concatenator process
func NewConcatenator(wf *scipipe.Workflow, name string, outPath string) *Concatenator {
p := &Concatenator{
BaseProcess: scipipe.NewBaseProcess(wf, name),
OutPath: outPath,
}
p.InitInPort(p, "in")
p.InitOutPort(p, "out")
wf.AddProc(p)
return p
}
// In returns the (only) in-port for this process
func (p *Concatenator) In() *scipipe.InPort { return p.InPort("in") }
// Out returns the (only) out-port for this process
func (p *Concatenator) Out() *scipipe.OutPort { return p.OutPort("out") }
// Run runs the Concatenator process
func (p *Concatenator) Run() {
defer p.CloseAllOutPorts()
outIP := scipipe.NewFileIP(p.OutPath)
outFh := outIP.OpenWriteTemp()
outIPsByTag := make(map[string]*scipipe.FileIP)
outFhsByTag := make(map[string]*os.File)
for inIP := range p.In().Chan {
tagVal := inIP.Tag(p.GroupByTag)
if tagVal != "" {
if _, ok := outIPsByTag[tagVal]; !ok {
outIPsByTag[tagVal] = scipipe.NewFileIP(fmt.Sprintf("%s.%s_%s", p.OutPath, p.GroupByTag, tagVal))
outIPsByTag[tagVal].AddTag(p.GroupByTag, tagVal)
outFhsByTag[tagVal] = outIPsByTag[tagVal].OpenWriteTemp()
}
dat, err := ioutil.ReadFile(inIP.Path())
scipipe.Check(err)
outFhsByTag[tagVal].Write(dat)
outFhsByTag[tagVal].Write([]byte("\n"))
} else {
dat, err := ioutil.ReadFile(inIP.Path())
scipipe.Check(err)
outFh.Write(dat)
outFh.Write([]byte("\n"))
}
}
// Close file handles
outFh.Close()
for _, taggedFh := range outFhsByTag {
taggedFh.Close()
}
// Send IPs
p.Out().Send(outIP)
for _, taggedIP := range outIPsByTag {
p.Out().Send(taggedIP)
}
}