/
main.go
123 lines (100 loc) · 2.68 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
package main
import (
"context"
"fmt"
"io"
"net/http"
"os"
"sync"
"golang.org/x/sync/semaphore"
"gopkg.in/yaml.v2"
)
type Repo struct {
URL string `yaml:"url"`
Version string `yaml:"version"`
Path string `yaml:"path"`
}
type Config struct {
OutputDir string `yaml:"output_dir"`
Repos map[string]Repo `yaml:"repos"`
}
var (
wg sync.WaitGroup // WaitGroup to wait for all goroutines to finish.
cache sync.Map // Cache to store and retrieve OpenAPI files.
)
func fetchFile(sema *semaphore.Weighted, repoName string, r Repo, outputDir string) {
defer wg.Done() // Notify WaitGroup that this goroutine is done.
defer sema.Release(1) // Release a spot in the semaphore.
url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s", r.URL, r.Version, r.Path)
// Check if the data is already in cache.
if v, ok := cache.Load(url); ok {
writeFile(repoName, r, outputDir, v.([]byte))
return
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
return
}
// If private repository, set necessary headers for authentication with GitHub token.
if username, token := os.Getenv("GITHUB_USERNAME"), os.Getenv("GITHUB_TOKEN"); username != "" && token != "" {
req.SetBasicAuth(username, token)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
if res.StatusCode != 200 {
fmt.Printf("Failed to fetch %s: %s\n", url, res.Status)
return
}
fileData, err := io.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
// Save the file data to the cache.
cache.Store(url, fileData)
writeFile(repoName, r, outputDir, fileData)
}
func writeFile(repoName string, r Repo, outputDir string, data []byte) {
destDir := fmt.Sprintf("%s/%s", outputDir, repoName)
err := os.MkdirAll(destDir, 0755)
if err != nil {
fmt.Println(err)
return
}
destFile := fmt.Sprintf("%s/%s.yaml", destDir, repoName)
err = os.WriteFile(destFile, data, 0644)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("Saved %s\n", destFile)
}
func main() {
data, err := os.ReadFile("oam.yaml")
if err != nil {
fmt.Println(err)
os.Exit(1)
}
var config Config
err = yaml.Unmarshal(data, &config)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
sema := semaphore.NewWeighted(20) // Semaphore to rate limit API calls.
for repoName, r := range config.Repos {
err := sema.Acquire(context.Background(), 1) // Grab a spot in the semaphore.
if err != nil {
fmt.Println(err)
continue
}
wg.Add(1) // Notify the WaitGroup that a new goroutine is starting.
go fetchFile(sema, repoName, r, config.OutputDir)
}
wg.Wait() // Wait for all goroutines to finish.
}