Bug 1825831: Refactoring collector, add Doc and doc generator

Makefile

@@ -16,6 +16,9 @@ vet:
 lint:
 	golint $$(go list ./... | grep -v /vendor/)
 
+gen-doc:
+	go run cmd/gendoc/main.go --out=docs/gathered-data.md
+
 vendor:
 	go mod tidy
 	go mod vendor

README.md

@@ -8,6 +8,10 @@ This cluster operator gathers anonymized system configuration and reports it to
 * ClusterOperator objects
 * All non-secret global config (hostnames and URLs anonymized)
 
+The list of all collected data with description, location in produced archive and link to Api and some examples is at [docs/gathered-data.md](docs/gathered-data.md)
+
+The resulting data is packed in .tar.gz archive with folder structure indicated in the document. Example of such archive is at [docs/insights-archive-sample](docs/insights-archive-sample).
+
 ## Building
 
 To build the operator, install Go 1.11 or above and run:
@@ -42,3 +46,13 @@ It is also possible to specify CLI options for Go test. For example, if you need
 
 Insights Operator is part of Red Hat OpenShift Container Platform. For product-related issues, please
 file a ticket [in Red Hat Bugzilla](https://bugzilla.redhat.com/enter_bug.cgi?product=OpenShift%20Container%20Platform&component=Insights%20Operator) for "Insights Operator" component.
+
+## Generating the document with gathered data
+The document docs/gathered-data contains list of collected data, the Api and some examples. The document is generated from package sources by looking for Gather... methods.
+If for any GatherXXX method exists its method which returns example with name ExampleXXX, the generated example is added to document with the size in bytes.
+
+
+To start generating the document run:
+```
+make gen-doc
+```

cmd/gendoc/main.go

@@ -0,0 +1,251 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+var (
+	expPkg     string
+	mdfname    string
+	mdf        *os.File
+	randSource = rand.NewSource(time.Now().UnixNano())
+	reGather   = regexp.MustCompile(`^(Gather)(.*)`)
+	reExample  = regexp.MustCompile(`^(Example)(.*)`)
+)
+
+func init() {
+	flag.StringVar(&expPkg, "gatherpkg", "gather", "Package where to find Gather methods")
+	flag.StringVar(&mdfname, "out", "gathered-data.md", "File to which MD doc will be generated")
+}
+
+type DocBlock struct {
+	Doc      string
+	Examples map[string]string
+}
+
+func main() {
+	flag.Parse()
+	var err error
+	mdf, err = os.Create(mdfname)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer mdf.Close()
+	cleanRoot := "./"
+
+	md := map[string]*DocBlock{}
+	err = walkDir(cleanRoot, md)
+	if err != nil {
+		fmt.Print(err)
+	}
+	// second pass will gather Sample..
+	err = walkDir(cleanRoot, md)
+	if err != nil {
+		fmt.Print(err)
+	}
+	keys := make([]string, 0, len(md))
+	for k := range md {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	mdf.WriteString(fmt.Sprintf("This document is auto-generated by `make gen-doc`\n\n"))
+	for _, k := range keys {
+		mdf.WriteString(fmt.Sprintf("## %s\n\n", k))
+		mdf.WriteString(fmt.Sprintf("%s\n\n", md[k].Doc))
+		if len(md[k].Examples) > 0 {
+			size := 0
+			for _, e := range md[k].Examples {
+				size = len(e)
+			}
+			size = size / len(md[k].Examples)
+			mdf.WriteString(fmt.Sprintf("Output raw size: %d\n\n", size))
+
+			mdf.WriteString(fmt.Sprintf("### Examples\n\n"))
+			for n, e := range md[k].Examples {
+				mdf.WriteString(fmt.Sprintf("#### %s\n", n))
+				mdf.WriteString(fmt.Sprintf("%s\n\n", e))
+			}
+		}
+	}
+	fmt.Printf("Done")
+}
+
+func walkDir(cleanRoot string, md map[string]*DocBlock) error {
+	expPath := ""
+	fset := token.NewFileSet() // positions are relative to fset
+	return filepath.Walk(cleanRoot, func(path string, info os.FileInfo, e1 error) error {
+		if !info.IsDir() {
+			return nil
+		}
+		if expPath != "" {
+			// filter only wanted path under our package
+			if !strings.Contains(path, expPath) {
+				return nil
+			}
+		}
+		d, err := parser.ParseDir(fset, path, nil, parser.ParseComments)
+		if err != nil {
+			fmt.Println(err)
+			return nil
+		}
+
+		for astPackageName, astPackage := range d {
+			if astPackageName != expPkg && expPath == "" {
+				continue
+			}
+			if expPath == "" && len(astPackage.Files) > 0 {
+				firstKey := ""
+				for key := range astPackage.Files {
+					firstKey = key
+					break
+				}
+				if firstKey != "" {
+					expPath = filepath.Dir(firstKey)
+				}
+			}
+
+			ast.Inspect(astPackage, func(n ast.Node) bool {
+				// handle function declarations
+				fn, ok := n.(*ast.FuncDecl)
+				if ok {
+					gatherMethodWithSuff := reGather.ReplaceAllString(fn.Name.Name, "$2")
+					_, ok2 := md[gatherMethodWithSuff]
+					if !ok2 && fn.Name.IsExported() && strings.HasPrefix(fn.Name.Name, "Gather") && len(fn.Name.Name) > len("Gather") {
+						doc := fn.Doc.Text()
+						md[gatherMethodWithSuff] = parseDoc(fn.Name.Name, doc)
+						fmt.Printf(fn.Name.Name + "\n")
+					}
+					// Example methods will have Example prefix, and might have additional case suffix:
+					// ExampleMostRecentMetrics_case1, we will remove Example prefix
+					exampleMethodWithSuff := reExample.ReplaceAllString(fn.Name.Name, "$2")
+					var gatherMethod = ""
+					for m := range md {
+						if strings.HasPrefix(exampleMethodWithSuff, m) {
+							gatherMethod = m
+							break
+						}
+					}
+
+					if gatherMethod != "" && fn.Name.IsExported() && strings.HasPrefix(fn.Name.Name, "Example") && len(fn.Name.Name) > len("Example") {
+
+						// Do not execute same method twice
+						_, ok := md[exampleMethodWithSuff].Examples[exampleMethodWithSuff]
+						if !ok {
+							methodFullpackage := getPackageName(cleanRoot, astPackage)
+
+							output, err := execExampleMethod(methodFullpackage, astPackageName, fn.Name.Name)
+							if err != nil {
+								fmt.Printf("Error when running Example in package %s method %s\n", methodFullpackage, fn.Name.Name)
+								fmt.Println(err)
+								fmt.Println(output)
+								return true
+							}
+							if md[exampleMethodWithSuff].Examples == nil {
+								md[exampleMethodWithSuff].Examples = map[string]string{}
+							}
+							md[exampleMethodWithSuff].Examples[exampleMethodWithSuff] = string(output)
+						}
+						fmt.Printf(fn.Name.Name + "\n")
+					}
+				}
+				return true
+			})
+		}
+		return nil
+	})
+}
+
+// getPackageName generates full package name from asp.Package
+//   astRoot the relative path where ast.Package was parsed from, because ast.Package is relative to astRoot path
+//   f ast.Package with containing files
+// The returned full package is inferred from files in ast.Package and GOPATH
+func getPackageName(astRoot string, f *ast.Package) string {
+	firstKey := ""
+	var mypkg string
+	for key := range f.Files {
+		firstKey = key
+		break
+	}
+	if firstKey != "" {
+		abspath, _ := filepath.Abs(filepath.Join(astRoot, filepath.Dir(firstKey)))
+
+		mypkg = strings.TrimPrefix(abspath, filepath.Join(goPath(), "src"))
+		mypkg = strings.TrimLeft(mypkg, string(filepath.Separator))
+	}
+	return mypkg
+}
+
+func goPath() string {
+	gopath := os.Getenv("GOPATH")
+	if gopath == "" {
+		gopath = build.Default.GOPATH
+	}
+	return gopath
+}
+
+// execExampleMethod executes the method by starting go run and capturing the produced standard output
+func execExampleMethod(methodFullPackage, methodPackage, methodName string) (string, error) {
+	f := createRandom()
+
+	tf, err := os.Create(f)
+	if err != nil {
+		fmt.Println(err)
+		return "", err
+	}
+	defer func() {
+		err := os.Remove(f)
+		if err != nil {
+			fmt.Print(err)
+		}
+	}()
+	tf.WriteString(fmt.Sprintf(`package main
+	import "%s"
+	import "fmt"
+
+	func main() { 
+		str, _ := %s.%s()
+		fmt.Print(str)
+	}
+		`, methodFullPackage, methodPackage, methodName))
+	tf.Close()
+	cmd := exec.Command("go", "run", "./"+f)
+	output, err := cmd.CombinedOutput()
+	return string(output), err
+}
+
+// createRandom creates a random non existing file name in current folder
+func createRandom() string {
+	var f string
+	for {
+		f = fmt.Sprintf("sample%d.go", randSource.Int63())
+		_, err := os.Stat(f)
+		if os.IsNotExist(err) {
+			break
+		}
+	}
+	return f
+}
+
+func parseDoc(method, doc string) *DocBlock {
+	if strings.HasPrefix(doc, method) {
+		doc = strings.TrimLeft(doc, method)
+	}
+	doc = strings.TrimLeft(doc, " ")
+
+	db := &DocBlock{Doc: doc}
+	return db
+}