This repository has been archived by the owner on Feb 8, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
definitions_sync.go
78 lines (67 loc) · 1.58 KB
/
definitions_sync.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
package main
import (
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/flarco/dbio/saas/airbyte"
"github.com/flarco/g"
"github.com/flarco/g/process"
"gopkg.in/yaml.v3"
)
func main() {
syncDefinitions(true)
}
// syncDefinitions will pull specs for all connectors
// and write to the specification folder
func syncDefinitions(force bool) {
//
connectors, err := airbyte.GetSourceConnectors(true)
g.LogFatal(err)
// run spec on all of them to get the cred properties
// then clean up
for _, c := range connectors {
g.Info("synching " + c.Definition.Name)
getSpecification(c, force)
}
}
func getSpecification(c airbyte.Connector, force bool) (err error) {
wd, _ := os.Getwd()
filePath := g.F(
"%s/specifications/%s.yaml",
filepath.Dir(wd),
c.Key(),
)
if !force && g.PathExists(filePath) {
return
}
p, _ := process.NewProc("docker")
cleanUp := func() {
// clean up image
err := p.Run("image", "rm", c.Definition.Image())
g.LogError(err, "could not remove image spec for: "+c.Definition.Name)
}
// pull image
err = c.Pull()
g.LogError(err, "could not pull image: "+c.Definition.Image())
if err != nil {
time.Sleep(5 * time.Second)
}
err = c.GetSpec()
if err != nil {
g.LogError(err, "could not get spec for: "+c.Definition.Name)
return
}
defer cleanUp()
specBytes, err := yaml.Marshal(c.Specification)
if err != nil {
g.LogError(err, "could not marshall spec for: "+c.Definition.Name)
return
}
err = ioutil.WriteFile(filePath, specBytes, 0755)
if err != nil {
g.LogError(err, "could not write spec for: "+c.Definition.Name)
return
}
return
}