generated from brevdev/seed
/
mergeshells.go
656 lines (578 loc) · 19.7 KB
/
mergeshells.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
//go:build !codeanalysis
package mergeshells
import (
"embed"
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"github.com/brevdev/brev-cli/pkg/collections"
breverrors "github.com/brevdev/brev-cli/pkg/errors"
"github.com/brevdev/brev-cli/pkg/files"
"github.com/brevdev/brev-cli/pkg/terminal"
"github.com/tidwall/gjson"
"golang.org/x/text/encoding/charmap"
)
//go:embed templates/*
var templateFs embed.FS
func GenerateShellScript(path string) string {
deps := GetDependencies(path)
// fmt.Println(strings.Join([]string{"** Recognized dependencies **", strings.Join(deps, " ")}, "\n"))
return DependenciesToShell("bash", deps...)
}
func DependenciesToShell(shell string, deps ...string) string {
shellString := strings.Join([]string{filepath.Join("#!/bin/", shell), MergeShells(deps...)}, "\n")
return shellString
}
func GetDependencies(path string) []string {
var deps []string
// to add a new recognizer for a language,
// write a method that returns *string,
// where nil means this does not recognize
// "" means use default
// and any non-empty string returned is a version number constraint for us to use
// then add it to the dictionary below under the same key
// as the folder you can find the installer within
// when looking under the templates/ directory.
// if there is or a pre-existing recognizer, append it to the end of the {} list
// if there is not a pre-existing recognizer, make certain to
// 1) add a folder of the same name under templates/
// 2) add any direct versions we support using their version number as file name
// 3) add a 'generic' fallback if we don't recognize a specific version, using key as filename
// TODO -- if there is no specific version installer for the version found
// then try to replace ${version} in the generic installer
// with the version number before importing it
supportedDependencyMap := map[string][]func(string) *string{
"node": {nodeVersion},
"gatsby": {gatsbyVersion},
"rust": {rustVersion},
"golang": {goVersion},
}
// these will be applied, in left-to-right order, to the version string returned by your version function
// before passing it to the finder / splicer of the install shell script for your dependency
processVersionMap := map[string][]func(string) string{
"golang": {transformGoVersion},
"default": {transformVersion},
}
for dep, fs := range supportedDependencyMap {
versions := collections.Filter(func(x *string) bool {
if x == nil {
return false
}
return true
}, collections.Fanout(fs, path))
if len(versions) > 0 {
version := versions[0]
if len(*version) > 0 {
versionTransforms := processVersionMap["default"]
if len(processVersionMap[dep]) > 0 {
versionTransforms = processVersionMap[dep]
}
finalVersion := collections.S(versionTransforms...)(*version)
dep = strings.Join([]string{dep, finalVersion}, "-")
}
deps = append(deps, dep)
}
}
return deps
}
// exists returns whether the given file or directory exists
func dirExists(path string) bool {
_, err := os.Stat(path)
if err == nil {
return true
}
if os.IsNotExist(err) {
return false
}
return false
}
type MergeShellStore interface {
GetFileAsString(path string) (string, error)
}
func ImportPath(t *terminal.Terminal, path string, store MergeShellStore) {
pathExists := dirExists(path)
if !pathExists {
fmt.Println(strings.Join([]string{"Path:", path, "does not exist."}, " "))
return
}
var gitpath string
if path == "." {
gitpath = filepath.Join(".git", "config")
} else {
gitpath = filepath.Join(path, ".git", "config")
}
file, error := store.GetFileAsString(gitpath)
if error != nil {
fmt.Println(strings.Join([]string{"Could not read .git/config at", path}, " "))
return
}
// Get GitUrl
var gitURL string
for _, v := range strings.Split(file, "\n") {
if strings.Contains(v, "url") {
gitURL = strings.Split(v, "= ")[1]
}
}
if len(gitURL) == 0 {
fmt.Println("no git url found")
return
}
if !dirExists(filepath.Join(path, ".brev", "setup.sh")) {
WriteBrevFile(t, GetDependencies(path), gitURL, path)
} else {
fmt.Println(".brev/setup.sh already exists - will not overwrite.")
}
}
func GenerateLogs(script string) string {
fragments := fromSh(script)
return strings.Join(collections.Fmap(extractInstallLine, fragments), "\n")
}
func extractInstallLine(fragment ShellFragment) string {
line := ""
if fragment.Comment != nil {
line = strings.Join([]string{"*", *fragment.Comment}, " ")
}
return line
}
func transformGoVersion(version string) string {
// in Ruby:
// `curl https://go.dev/dl/`.scan(/id=\"go([\w\.]+)\"/).flat_map{|x| x}.filter{|x| x.start_with?(version)}[0]
if strings.Count(version, ".") < 2 {
// if this is not a well-specified version
// then get the list of versions from the official go website
out, err := exec.Command("curl", "https://go.dev/dl/").Output()
if err != nil {
fmt.Println("Issue fetching go version, defaulting to same as input")
return version
}
value := string(out)
// pull out the go versions from the ids of the subsections of the download page
re := regexp.MustCompile("id=\"go([\\w\\.]+)\"")
versions := re.FindAllString(value, 1000)
versions = collections.Fmap(func(x string) string { return x[6 : len(x)-1] }, versions)
prefixFn := collections.P2(collections.Flip(strings.HasPrefix), version)
// and fetch the first version that matches the prefix version you are looking for
// as it is rendered on the go page in reverse chronological order
// of nearest release to farthest for each version
matchingVersion := collections.First(collections.Filter(prefixFn, versions))
if matchingVersion == nil {
fmt.Println(strings.Join([]string{"No stable version found for", version}, " "))
// no stable version found so return the original version
return version
}
return *matchingVersion
}
return version
}
func transformVersion(version string) string {
if len(version) > 0 {
switch version[0:0] {
case "~":
return version[1:]
case "^":
return version[1:]
case ">":
if version[0:1] == ">=" {
return version[2:]
}
return version[1:]
case "<":
if version[0:1] == "<=" {
return version[2:]
}
return version[1:]
case "=":
return version[1:]
}
return version
}
return version
}
func WriteBrevFile(t *terminal.Terminal, deps []string, gitURL string, path string) *error {
fmt.Println("\n\nGitUrl: ", gitURL)
fmt.Println("** Found Dependencies **")
t.Vprint(t.Yellow(strings.Join(deps, " \n")))
shellString := GenerateShellScript(path)
fmt.Println(GenerateLogs(shellString))
mderr := os.MkdirAll(filepath.Join(path, ".brev"), os.ModePerm)
if mderr == nil {
// generate a string that is the collections.Concatenation of dependency-ordering the contents of all the dependencies
// found by cat'ing the directory generated from the deps string, using the translated ruby code with go generics
// place within .brev and write setup.sh with the result of this
f, err := os.Create(filepath.Join(path, ".brev", "setup.sh"))
if err != nil {
return &err
}
defer f.Close()
f.WriteString(shellString)
f.Sync()
} else {
return &mderr
}
return nil
}
type ShellFragment struct {
Name *string `json:"name"`
Tag *string `json:"tag"`
Comment *string `json:"comment"`
Script []string `json:"script"`
Dependencies []string `json:"dependencies"`
}
func parseCommentLine(line string) ShellFragment {
commentFreeLine := strings.TrimSpace(strings.ReplaceAll(line, "#", ""))
parts := strings.Split(commentFreeLine, " ")
if strings.HasPrefix(parts[0], "dependencies") {
temp := parts[1:]
return ShellFragment{Dependencies: temp}
}
if len(parts) == 2 {
return ShellFragment{Name: &parts[0], Tag: &parts[1]}
} else if len(parts) == 1 {
return ShellFragment{Name: &parts[0]}
} else {
return ShellFragment{Comment: &commentFreeLine}
}
}
func toDefsDict(shFragment []ShellFragment) map[string]ShellFragment {
return collections.Foldl(func(acc map[string]ShellFragment, frag ShellFragment) map[string]ShellFragment { //nolint:typecheck
if frag.Name != nil {
acc[*frag.Name] = frag
}
return acc
}, map[string]ShellFragment{}, shFragment)
}
type scriptAccumulator struct {
CurrentFragment ShellFragment
ScriptSoFar []ShellFragment
}
func fromSh(shScript string) []ShellFragment {
// get lines
lines := strings.Split(shScript, "\n")
base := scriptAccumulator{}
// collections.Foldl lines pulling out parsable information and occasionally pushing onto script so far and generating a new fragment, if need be
acc := collections.Foldl(func(acc scriptAccumulator, line string) scriptAccumulator { //nolint:typecheck
if strings.HasPrefix(line, "#") { // then it is a comment string, which may or may not have parsable information
if len(acc.CurrentFragment.Script) > 0 {
acc.ScriptSoFar = append(acc.ScriptSoFar, acc.CurrentFragment)
acc.CurrentFragment = ShellFragment{}
}
parsed := parseCommentLine(line)
if parsed.Name != nil {
acc.CurrentFragment.Name = parsed.Name
}
if parsed.Tag != nil {
acc.CurrentFragment.Tag = parsed.Tag
}
if parsed.Comment != nil {
acc.CurrentFragment.Comment = parsed.Comment
}
if parsed.Dependencies != nil {
acc.CurrentFragment.Dependencies = parsed.Dependencies
}
} else {
acc.CurrentFragment.Script = append(acc.CurrentFragment.Script, line)
}
return acc
}, base, lines)
// return the script so far plus the last fragment remaining after folding
return append(acc.ScriptSoFar, acc.CurrentFragment)
}
func importFile(nameVersion string) ([]ShellFragment, error) {
// split the name string into two with - -- left hand side is package, right hand side is version
// read from the generated path
// generate ShellFragment from it (fromSh) and return it
subPaths := strings.Split(nameVersion, "-")
noversion := false
if len(subPaths) == 1 {
noversion = true
subPaths = collections.Duplicate(subPaths[0])
}
path := filepath.Join(collections.Concat([]string{"templates"}, subPaths)...) //nolint:typecheck
script, err := templateFs.Open(path)
if err != nil && !noversion {
if subPaths[0] != subPaths[1] {
path = filepath.Join(collections.Concat([]string{"templates"}, collections.Duplicate(subPaths[0]))...) //nolint:typecheck
script, err = templateFs.Open(path)
if err != nil {
return []ShellFragment{}, err
}
} else {
return []ShellFragment{}, errors.New(strings.Join([]string{"Path does not exist:", path}, " "))
}
}
out, err := ioutil.ReadAll(script)
stringScript := string(out)
if !noversion {
stringScript = strings.ReplaceAll(stringScript, "${version}", subPaths[1])
}
// fmt.Println(stringScript)
if err != nil {
return []ShellFragment{}, err
}
return fromSh(stringScript), nil
}
func toSh(script []ShellFragment) string {
// collections.Flatmap across generating the script from all of the component shell bits
return strings.Join(collections.Flatmap(func(frag ShellFragment) []string { //nolint:typecheck
name, tag, comment := frag.Name, frag.Tag, frag.Comment
innerScript, dependencies := frag.Script, frag.Dependencies
returnval := []string{}
if name != nil {
nameTag := strings.Join([]string{"#", *name}, " ")
if tag != nil {
nameTag = strings.Join([]string{nameTag, *tag}, " ")
}
returnval = append(returnval, nameTag)
}
if len(dependencies) > 0 {
returnval = append(returnval, strings.Join([]string{"# dependencies:", strings.Join(dependencies, " ")}, " "))
}
if comment != nil {
returnval = append(returnval, strings.Join([]string{"#", *comment}, " "))
}
returnval = append(returnval, strings.Join(innerScript, "\n"))
return returnval
}, script), "\n")
}
func findDependencies(shName string, baselineDependencies map[string]ShellFragment, globalDependencies map[string]ShellFragment) []string {
definition := ShellFragment{}
if val, ok := baselineDependencies[shName]; ok {
definition = val
} else if val2, ok2 := globalDependencies[shName]; ok2 {
definition = val2
}
dependencies := definition.Dependencies
return collections.Flatmap(func(dep_name string) []string { //nolint:typecheck
return append(findDependencies(dep_name, baselineDependencies, globalDependencies), dep_name)
}, dependencies)
}
func splitIntoLibraryAndSeq(shFragments []ShellFragment) ([]string, map[string]ShellFragment, []string) {
return collections.Fmap(func(some ShellFragment) string { //nolint:typecheck
if some.Name != nil {
return *some.Name
} else {
return ""
}
}, shFragments), toDefsDict(shFragments), []string{}
}
func prependDependencies(shName string, baselineDependencies map[string]ShellFragment, globalDependencies map[string]ShellFragment) OrderDefsFailures {
dependencies := collections.Uniq(findDependencies(shName, baselineDependencies, globalDependencies)) //nolint:typecheck
// baseline_deps := collections.Filter(func (dep string) bool {
// if _, ok := baselineDependencies[dep]; ok {
// return true
// }
// return false
// }, dependencies)
nonBaselineDependencies := collections.Filter(func(dep string) bool { //nolint:typecheck
if _, ok := baselineDependencies[dep]; ok {
return false
}
return true
}, dependencies)
canBeFixedDependencies := collections.Filter(func(dep string) bool { //nolint:typecheck
if _, ok := globalDependencies[dep]; ok {
return true
}
return false
}, nonBaselineDependencies)
cantBeFixedDependencies := collections.Difference(nonBaselineDependencies, canBeFixedDependencies) //nolint:typecheck
addedBaselineDependencies := collections.Foldl( //nolint:typecheck
func(deps map[string]ShellFragment, dep string) map[string]ShellFragment {
deps[dep] = globalDependencies[dep]
return deps
}, map[string]ShellFragment{}, canBeFixedDependencies)
return OrderDefsFailures{Order: append(dependencies, shName), Defs: collections.DictMerge(addedBaselineDependencies, baselineDependencies), Failures: cantBeFixedDependencies} //nolint:typecheck
}
type OrderDefsFailures struct {
Order []string
Defs map[string]ShellFragment
Failures []string
}
func addDependencies(shFragments []ShellFragment, baselineDependencies map[string]ShellFragment, globalDependencies map[string]ShellFragment) OrderDefsFailures {
// ## it's a left fold across the importFile statements
// ## at any point, if the dependencies aren't already in the loaded dictionary
// ## we add them before we add the current item, and we add them and the current item into the loaded dictionary
order, shellDefs, failures := splitIntoLibraryAndSeq(shFragments)
newestOrderDefsFailures := collections.Foldl(func(acc OrderDefsFailures, shName string) OrderDefsFailures { //nolint:typecheck
newOrderDefsFailures := prependDependencies(shName, acc.Defs, globalDependencies)
return OrderDefsFailures{Order: append(collections.Concat(acc.Order, newOrderDefsFailures.Order), shName), Failures: collections.Concat(acc.Failures, newOrderDefsFailures.Failures), Defs: collections.DictMerge(acc.Defs, newOrderDefsFailures.Defs)} //nolint:typecheck
}, OrderDefsFailures{Order: []string{}, Failures: []string{}, Defs: shellDefs}, order)
return OrderDefsFailures{Order: collections.Uniq(newestOrderDefsFailures.Order), Defs: newestOrderDefsFailures.Defs, Failures: collections.Uniq(collections.Concat(failures, newestOrderDefsFailures.Failures))} //nolint:typecheck
}
func process(shFragments []ShellFragment, baselineDependencies map[string]ShellFragment, globalDependencies map[string]ShellFragment) []ShellFragment {
resultOrderDefsFailures := addDependencies(shFragments, baselineDependencies, globalDependencies)
// TODO print or return the failed installation instructions "FAILED TO FIND INSTALLATION INSTRUCTIONS FOR: #{failures}" if failures.length > 0
return collections.Fmap(func(x string) ShellFragment { //nolint:typecheck
return resultOrderDefsFailures.Defs[x]
}, resultOrderDefsFailures.Order)
}
func MergeShells(deps ...string) string {
importedDependencies := collections.Flatmap(func(dep string) []ShellFragment { //nolint:typecheck
frags, err := importFile(dep)
if err != nil {
return []ShellFragment{}
} else {
return frags
}
}, deps)
processedDependencies := process(importedDependencies, map[string]ShellFragment{}, map[string]ShellFragment{})
shellScript := toSh(processedDependencies)
return shellScript
}
func rustVersion(path string) *string {
paths := recursivelyFindFile([]string{"Cargo\\.toml", "Cargo\\.lock"}, path)
if len(paths) > 0 {
retval := ""
return &retval
}
return nil
}
func nodeVersion(path string) *string {
paths := recursivelyFindFile([]string{"package\\-lock\\.json$", "package\\.json$"}, path)
retval := ""
if len(paths) > 0 {
sort.Strings(paths)
i := len(paths) - 1
keypath := "engines.node"
jsonstring, err := files.CatFile(paths[i])
value := gjson.Get(jsonstring, "name")
value = gjson.Get(jsonstring, keypath)
if err != nil {
//
}
if retval == "" {
retval = value.String()
}
return &retval
}
return nil
}
func gatsbyVersion(path string) *string {
paths := recursivelyFindFile([]string{"package\\.json"}, path)
retval := ""
var foundGatsby bool
if len(paths) > 0 {
sort.Strings(paths)
for _, path := range paths {
keypath := "dependencies.gatsby"
jsonstring, err := files.CatFile(path)
value := gjson.Get(jsonstring, keypath)
if err != nil {
//
}
if value.String() != "" {
foundGatsby = true
}
}
if foundGatsby {
return &retval
}
return nil
}
return nil
}
func goVersion(path string) *string {
paths := recursivelyFindFile([]string{"go\\.mod"}, path)
if len(paths) > 0 {
sort.Strings(paths)
for _, path := range paths {
fmt.Println(path)
res, err := readGoMod(path)
if err != nil {
//
}
return &res
}
}
return nil
}
func isRuby(path string) bool {
paths := recursivelyFindFile([]string{"Gemfile.lock", "Gemfile"}, path)
if len(paths) > 0 {
return true
}
return false
}
func isPython(path string) bool {
paths := recursivelyFindFile([]string{"Gemfile.lock", "Gemfile"}, path)
if len(paths) > 0 {
return true
}
return false
}
func appendPath(a string, b string) string {
if a == "." {
return b
}
return a + "/" + b
}
// Returns list of paths to file
func recursivelyFindFile(filenames []string, path string) []string {
var paths []string
files, err := ioutil.ReadDir(path)
if err != nil {
fmt.Println(err)
}
for _, f := range files {
dir, err := os.Stat(appendPath(path, f.Name()))
if err != nil {
// fmt.Println(t.Red(err.Error()))
} else {
for _, filename := range filenames {
r, _ := regexp.Compile(filename)
res := r.MatchString(f.Name())
if res {
// t.Vprint(t.Yellow(filename) + "---" + t.Yellow(path+f.Name()))
paths = append(paths, appendPath(path, f.Name()))
// fileContents, err := catFile(appendPath(path, f.Name()))
// if err != nil {
// //
// }
// TODO: read
// if file has json, read the json
}
}
if dir.IsDir() {
paths = append(paths, recursivelyFindFile(filenames, appendPath(path, f.Name()))...)
}
}
}
// TODO: make the list collections.Unique
return paths
}
//
// read from gomod
// read from json
func CatFile(filePath string) (string, error) {
gocmd := exec.Command("cat", filePath) // #nosec G204
in, err := gocmd.Output()
if err != nil {
return "", breverrors.WrapAndTrace(err, "error reading file "+filePath)
} else {
d := charmap.CodePage850.NewDecoder()
out, err := d.Bytes(in)
if err != nil {
return "", breverrors.WrapAndTrace(err, "error reading file "+filePath)
}
return string(out), nil
}
}
func readGoMod(filePath string) (string, error) {
contents, err := files.CatFile(filePath)
if err != nil {
return "", err
}
lines := strings.Split(contents, "\n")
for _, line := range lines {
if strings.Contains(line, "go ") {
return strings.Split(line, " ")[1], nil
}
}
return "", nil
}