/
dependencyverifier.go
391 lines (358 loc) · 12.7 KB
/
dependencyverifier.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"sort"
"strings"
"github.com/google/go-cmp/cmp"
)
type Unwanted struct {
// things we want to stop referencing
Spec UnwantedSpec `json:"spec"`
// status of our unwanted dependencies
Status UnwantedStatus `json:"status"`
}
type UnwantedSpec struct {
// module names we don't want to depend on, mapped to an optional message about why
UnwantedModules map[string]string `json:"unwantedModules"`
}
type UnwantedStatus struct {
// references to modules in the spec.unwantedModules list, based on `go mod graph` content.
// eliminating things from this list is good, and sometimes requires working with upstreams to do so.
UnwantedReferences map[string][]string `json:"unwantedReferences"`
// list of modules in the spec.unwantedModules list which are vendored
UnwantedVendored []string `json:"unwantedVendored"`
}
// runCommand runs the cmd and returns the combined stdout and stderr, or an
// error if the command failed.
func runCommand(cmd ...string) (string, error) {
return runCommandInDir("", cmd)
}
func runCommandInDir(dir string, cmd []string) (string, error) {
c := exec.Command(cmd[0], cmd[1:]...)
c.Dir = dir
output, err := c.CombinedOutput()
if err != nil {
return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
}
return string(output), nil
}
func readFile(path string) (string, error) {
content, err := os.ReadFile(path)
// Convert []byte to string and print to screen
return string(content), err
}
func moduleInSlice(a module, list []module, matchVersion bool) bool {
for _, b := range list {
if b == a {
return true
}
if !matchVersion && b.name == a.name {
return true
}
}
return false
}
// converts `go mod graph` output modStr into a map of from->[]to references and the main module
func convertToMap(modStr string) ([]module, map[module][]module) {
var (
mainModulesList = []module{}
mainModules = map[module]bool{}
)
modMap := make(map[module][]module)
for _, line := range strings.Split(modStr, "\n") {
if len(line) == 0 {
continue
}
deps := strings.Split(line, " ")
if len(deps) == 2 {
first := parseModule(deps[0])
second := parseModule(deps[1])
if first.version == "" || first.version == "v0.0.0" {
if !mainModules[first] {
mainModules[first] = true
mainModulesList = append(mainModulesList, first)
}
}
modMap[first] = append(modMap[first], second)
} else {
// skip invalid line
log.Printf("!!!invalid line in mod.graph: %s", line)
continue
}
}
return mainModulesList, modMap
}
// difference returns a-b and b-a as sorted lists
func difference(a, b []string) ([]string, []string) {
aMinusB := map[string]bool{}
bMinusA := map[string]bool{}
for _, dependency := range a {
aMinusB[dependency] = true
}
for _, dependency := range b {
if _, found := aMinusB[dependency]; found {
delete(aMinusB, dependency)
} else {
bMinusA[dependency] = true
}
}
aMinusBList := []string{}
bMinusAList := []string{}
for dependency := range aMinusB {
aMinusBList = append(aMinusBList, dependency)
}
for dependency := range bMinusA {
bMinusAList = append(bMinusAList, dependency)
}
sort.Strings(aMinusBList)
sort.Strings(bMinusAList)
return aMinusBList, bMinusAList
}
type module struct {
name string
version string
}
func (m module) String() string {
if len(m.version) == 0 {
return m.name
}
return m.name + "@" + m.version
}
func parseModule(s string) module {
if !strings.Contains(s, "@") {
return module{name: s}
}
parts := strings.SplitN(s, "@", 2)
return module{name: parts[0], version: parts[1]}
}
// option1: dependencyverifier dependencies.json
// it will run `go mod graph` and check it.
func main() {
var modeGraphStr string
var err error
if len(os.Args) == 2 {
// run `go mod graph`
modeGraphStr, err = runCommand("go", "mod", "graph")
if err != nil {
log.Fatalf("Error running 'go mod graph': %s", err)
}
} else {
log.Fatalf("Usage: %s dependencies.json", os.Args[0])
}
dependenciesJSONPath := string(os.Args[1])
dependencies, err := readFile(dependenciesJSONPath)
if err != nil {
log.Fatalf("Error reading dependencies file %s: %s", dependencies, err)
}
// load Unwanted from json
configFromFile := &Unwanted{}
decoder := json.NewDecoder(bytes.NewBuffer([]byte(dependencies)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(configFromFile); err != nil {
log.Fatalf("Error reading dependencies file %s: %s", dependenciesJSONPath, err)
}
// convert from `go mod graph` to main module and map of from->[]to references
mainModules, moduleGraph := convertToMap(modeGraphStr)
directDependencies := map[string]map[string]bool{}
for _, mainModule := range mainModules {
dir := ""
if mainModule.name != "k8s.io/kubernetes" {
dir = "staging/src/" + mainModule.name
}
listOutput, err := runCommandInDir(dir, []string{"go", "list", "-m", "-f", "{{if not .Indirect}}{{if not .Main}}{{.Path}}{{end}}{{end}}", "all"})
if err != nil {
log.Fatalf("Error running 'go list' for %s: %s", mainModule.name, err)
}
directDependencies[mainModule.name] = map[string]bool{}
for _, directDependency := range strings.Split(listOutput, "\n") {
directDependencies[mainModule.name][directDependency] = true
}
}
// gather the effective versions by looking at the versions required by the main modules
effectiveVersions := map[string]module{}
for _, mainModule := range mainModules {
for _, override := range moduleGraph[mainModule] {
if _, ok := effectiveVersions[override.name]; !ok {
effectiveVersions[override.name] = override
}
}
}
unwantedToReferencers := map[string][]module{}
for _, mainModule := range mainModules {
// visit to find unwanted modules still referenced from the main module
visit(func(m module, via []module) {
if _, unwanted := configFromFile.Spec.UnwantedModules[m.name]; unwanted {
// this is unwanted, store what is referencing it
referencer := via[len(via)-1]
if !moduleInSlice(referencer, unwantedToReferencers[m.name], false) {
// // uncomment to get a detailed tree of the path that referenced the unwanted dependency
//
// i := 0
// for _, v := range via {
// if v.version != "" && v.version != "v0.0.0" {
// fmt.Println(strings.Repeat(" ", i), v)
// i++
// }
// }
// if i > 0 {
// fmt.Println(strings.Repeat(" ", i+1), m)
// fmt.Println()
// }
unwantedToReferencers[m.name] = append(unwantedToReferencers[m.name], referencer)
}
}
}, mainModule, moduleGraph, effectiveVersions)
}
config := &Unwanted{}
config.Spec.UnwantedModules = configFromFile.Spec.UnwantedModules
for unwanted := range unwantedToReferencers {
if config.Status.UnwantedReferences == nil {
config.Status.UnwantedReferences = map[string][]string{}
}
sort.Slice(unwantedToReferencers[unwanted], func(i, j int) bool {
ri := unwantedToReferencers[unwanted][i]
rj := unwantedToReferencers[unwanted][j]
if ri.name != rj.name {
return ri.name < rj.name
}
return ri.version < rj.version
})
for _, referencer := range unwantedToReferencers[unwanted] {
// make sure any reference at all shows up as a non-nil status
if config.Status.UnwantedReferences == nil {
config.Status.UnwantedReferences[unwanted] = []string{}
}
// record specific names of versioned referents
if referencer.version != "" && referencer.version != "v0.0.0" {
config.Status.UnwantedReferences[unwanted] = append(config.Status.UnwantedReferences[unwanted], referencer.name)
} else if directDependencies[referencer.name][unwanted] {
config.Status.UnwantedReferences[unwanted] = append(config.Status.UnwantedReferences[unwanted], referencer.name)
}
}
}
vendorModulesTxt, err := ioutil.ReadFile("vendor/modules.txt")
if err != nil {
log.Fatal(err)
}
vendoredModules := map[string]bool{}
for _, l := range strings.Split(string(vendorModulesTxt), "\n") {
parts := strings.Split(l, " ")
if len(parts) == 3 && parts[0] == "#" && strings.HasPrefix(parts[2], "v") {
vendoredModules[parts[1]] = true
}
}
config.Status.UnwantedVendored = []string{}
for unwanted := range configFromFile.Spec.UnwantedModules {
if vendoredModules[unwanted] {
config.Status.UnwantedVendored = append(config.Status.UnwantedVendored, unwanted)
}
}
sort.Strings(config.Status.UnwantedVendored)
needUpdate := false
// Compare unwanted list from unwanted-dependencies.json with current status from `go mod graph`
expected, err := json.MarshalIndent(configFromFile.Status, "", " ")
if err != nil {
log.Fatal(err)
}
actual, err := json.MarshalIndent(config.Status, "", " ")
if err != nil {
log.Fatal(err)
}
if !bytes.Equal(expected, actual) {
log.Printf("Expected status of\n%s", string(expected))
log.Printf("Got status of\n%s", string(actual))
needUpdate = true
log.Print("Status diff:\n", cmp.Diff(expected, actual))
}
for expectedRef, expectedFrom := range configFromFile.Status.UnwantedReferences {
actualFrom, ok := config.Status.UnwantedReferences[expectedRef]
if !ok {
// disappeared entirely
log.Printf("Good news! Unwanted dependency %q is no longer referenced. Remove status.unwantedReferences[%q] in %s to ensure it doesn't get reintroduced.", expectedRef, expectedRef, dependenciesJSONPath)
needUpdate = true
continue
}
removedReferences, unwantedReferences := difference(expectedFrom, actualFrom)
if len(removedReferences) > 0 {
log.Printf("Good news! Unwanted module %q dropped the following dependants:", expectedRef)
for _, reference := range removedReferences {
log.Printf(" %s", reference)
}
log.Printf("!!! Remove those from status.unwantedReferences[%q] in %s to ensure they don't get reintroduced.", expectedRef, dependenciesJSONPath)
needUpdate = true
}
if len(unwantedReferences) > 0 {
log.Printf("Unwanted module %q marked in %s is referenced by new dependants:", expectedRef, dependenciesJSONPath)
for _, reference := range unwantedReferences {
log.Printf(" %s", reference)
}
log.Printf("!!! Avoid updating referencing modules to versions that reintroduce use of unwanted dependencies\n")
needUpdate = true
}
}
for actualRef, actualFrom := range config.Status.UnwantedReferences {
if _, expected := configFromFile.Status.UnwantedReferences[actualRef]; expected {
// expected, already ensured referencers were equal in the first loop
continue
}
log.Printf("Unwanted module %q marked in %s is referenced", actualRef, dependenciesJSONPath)
for _, reference := range actualFrom {
log.Printf(" %s", reference)
}
log.Printf("!!! Avoid updating referencing modules to versions that reintroduce use of unwanted dependencies\n")
needUpdate = true
}
removedVendored, addedVendored := difference(configFromFile.Status.UnwantedVendored, config.Status.UnwantedVendored)
if len(removedVendored) > 0 {
log.Printf("Good news! Unwanted modules are no longer vendered: %q", removedVendored)
log.Printf("!!! Remove those from status.unwantedVendored in %s to ensure they don't get reintroduced.", dependenciesJSONPath)
needUpdate = true
}
if len(addedVendored) > 0 {
log.Printf("Unwanted modules are newly vendored: %q", addedVendored)
log.Printf("!!! Avoid updates that increase vendoring of unwanted dependencies\n")
needUpdate = true
}
if needUpdate {
os.Exit(1)
}
}
func visit(visitor func(m module, via []module), main module, references map[module][]module, effectiveVersions map[string]module) {
doVisit(visitor, main, nil, map[module]bool{}, references, effectiveVersions)
}
func doVisit(visitor func(m module, via []module), from module, via []module, visited map[module]bool, references map[module][]module, effectiveVersions map[string]module) {
visitor(from, via)
via = append(via, from)
if visited[from] {
return
}
for _, to := range references[from] {
// switch to the effective version of this dependency
if override, ok := effectiveVersions[to.name]; ok {
to = override
}
// recurse unless we've already visited this module in this traversal
if !moduleInSlice(to, via, false) {
doVisit(visitor, to, via, visited, references, effectiveVersions)
}
}
visited[from] = true
}