streamline Maobi

build.sh

@@ -4,14 +4,19 @@ die() { echo "$*" 1>&2 ; exit 1; }
 VERSION="v$(cat version)-$(date "+%Y%m%d")"
 REPO=$(basename "$(dirname "$(pwd)")")/$(basename "$(pwd)")
 LDFLAGS="-X main.version=$VERSION -X main.repo=$REPO"
+TAG="simagix/keyhole"
 [[ "$(which go)" = "" ]] && die "go command not found"
 [[ "$GOPATH" = "" ]] && die "GOPATH not set"
 [[ "${GOPATH}/src/github.com/$REPO" != "$(pwd)" ]] && die "building keyhole should be under ${GOPATH}/src/github.com/$REPO"
 mkdir -p dist
-if [[ "$1" == "all" ]]; then
+if [[ "$1" == "docker" ]]; then
   docker rmi -f $(docker images -f "dangling=true" -q) > /dev/null 2>&1
-  docker build  -f Dockerfile . -t $REPO
-  id=$(docker create $REPO)
+  BR=$(git branch --show-current)
+  if [[ "${BR}" == "master" ]]; then
+    BR="latest"
+  fi 
+  docker build  -f Dockerfile -t ${TAG}:${BR} .
+  id=$(docker create ${TAG}:${BR})
   docker cp $id:/dist - | tar vx
 else
   if [ "$1" == "cross-platform"  ]; then

cluster_info.go

@@ -15,14 +15,16 @@ func GetClusterSummary(version string, client *mongo.Client) string {
 }
 
 // CollectCluserDetails collects cluster details
-func CollectCluserDetails(stats *mdb.ClusterStats, client *mongo.Client, connString connstring.ConnString) error {
+func CollectCluserDetails(stats *mdb.ClusterStats, client *mongo.Client, connString connstring.ConnString, maobiURL string) error {
 	var err error
+	var data []byte
+	var ofile string
 	if err = stats.GetClusterStats(client, connString); err != nil {
 		return err
 	}
-	if err = stats.OutputBSON(); err != nil {
+	stats.Print()
+	if ofile, data, err = stats.OutputBSON(); err != nil {
 		return err
 	}
-	stats.Print()
-	return err
+	return GenerateMaobiReport(maobiURL, data, ofile)
 }

comparison.go

@@ -192,7 +192,6 @@ func (p *Comparison) OutputBSON() error {
 	if data, err = bson.Marshal(p); err != nil {
 		return err
 	}
-	outdir := "./out"
 	os.Mkdir(outdir, 0755)
 	basename := p.TargetStats.HostInfo.System.Hostname
 	basename = strings.ReplaceAll(basename, ":", "_")

indexes.go

@@ -25,15 +25,17 @@ func DuplicateIndexesFromFile(index *mdb.IndexStats, client *mongo.Client, filen
 }
 
 // CollectIndexStats collects all indexes stats
-func CollectIndexStats(index *mdb.IndexStats, client *mongo.Client) error {
+func CollectIndexStats(index *mdb.IndexStats, client *mongo.Client, maobiURL string) error {
 	var err error
+	var data []byte
+	var ofile string
 	var databases []mdb.Database
 	if databases, err = index.GetIndexes(client); err != nil {
 		return err
 	}
 	index.PrintIndexesOf(databases)
-	if err = index.OutputBSON(); err != nil {
+	if ofile, data, err = index.OutputBSON(); err != nil {
 		return err
 	}
-	return err
+	return GenerateMaobiReport(maobiURL, data, ofile)
 }

keyhole.go

@@ -21,10 +21,15 @@ import (
 	"go.mongodb.org/mongo-driver/x/mongo/driver/connstring"
 )
 
+const (
+	outdir  = "./out"
+	htmldir = "./html"
+)
+
 // Run executes main()
 func Run(fullVersion string) {
 	var err error
-	allinfo := flag.Bool("allinfo", false, "get all cluster info")
+	allinfo := flag.String("allinfo", "", "database connection string")
 	cardinality := flag.String("cardinality", "", "check collection cardinality")
 	changeStreams := flag.Bool("changeStreams", false, "change streams watch")
 	collection := flag.String("collection", "", "collection name to print schema")
@@ -38,9 +43,10 @@ func Run(fullVersion string) {
 	explain := flag.String("explain", "", "explain a query from a JSON doc or a log line")
 	file := flag.String("file", "", "template file for seedibg data")
 	ftdc := flag.Bool("ftdc", false, "download from atlas://user:key@group/cluster")
-	index := flag.Bool("index", false, "get indexes info")
-	info := flag.Bool("info", false, "Atlas info (atlas://user:key)")
+	index := flag.String("index", "", "get indexes info")
+	info := flag.String("info", "", "database connection string (Atlas uses atlas://user:key)")
 	loginfo := flag.Bool("loginfo", false, "log performance analytic from file or Atlas")
+	maobiURL := flag.String("maobi", "", "maobi url")
 	nocolor := flag.Bool("nocolor", false, "disable color codes")
 	pause := flag.Bool("pause", false, "pause an Atlas cluster atlas://user:key@group/cluster")
 	peek := flag.Bool("peek", false, "only collect stats")
@@ -68,7 +74,13 @@ func Run(fullVersion string) {
 	flagset := make(map[string]bool)
 	flag.Visit(func(f *flag.Flag) { flagset[f.Name] = true })
 	var uri string
-	if len(flag.Args()) > 0 {
+	if *allinfo != "" {
+		uri = *allinfo
+	} else if *info != "" {
+		uri = *info
+	} else if *index != "" {
+		uri = *index
+	} else if len(flag.Args()) > 0 {
 		uri = flag.Arg(0)
 	}
 
@@ -79,7 +91,9 @@ func Run(fullVersion string) {
 		}
 		api.SetArgs(flag.Args())
 		api.SetFTDC(*ftdc)
-		api.SetInfo(*info)
+		if *info != "" {
+			api.SetInfo(true)
+		}
 		api.SetLoginfo(*loginfo)
 		api.SetPause(*pause)
 		api.SetResume(*resume)
@@ -94,7 +108,7 @@ func Run(fullVersion string) {
 			l.SetRegexPattern(*regex)
 			l.SetSilent(*nocolor)
 			l.SetVerbose(*verbose)
-			if err = AnalyzeMongoLogs(l, api.GetLogNames()); err != nil {
+			if err = AnalyzeMongoLogs(l, api.GetLogNames(), *maobiURL); err != nil {
 				log.Fatal(err)
 			}
 		}
@@ -148,7 +162,7 @@ func Run(fullVersion string) {
 		l.SetRegexPattern(*regex)
 		l.SetSilent(*nocolor)
 		l.SetVerbose(*verbose)
-		if err = AnalyzeMongoLogs(l, flag.Args()); err != nil {
+		if err = AnalyzeMongoLogs(l, flag.Args(), *maobiURL); err != nil {
 			log.Fatal(err)
 		}
 		return
@@ -162,7 +176,7 @@ func Run(fullVersion string) {
 	} else if *viewlog != "" {
 		mdb.OutputLogInOldFormat(*viewlog)
 		return
-	} else if len(flag.Args()) == 0 {
+	} else if uri == "" {
 		flag.PrintDefaults()
 		fmt.Println("\nusage: keyhole [options] <connection_string>")
 		return
@@ -179,12 +193,12 @@ func Run(fullVersion string) {
 		log.Fatal(err)
 	}
 
-	if *allinfo || (*info && *verbose) {
+	if *allinfo != "" || (*info != "" && *verbose) {
 		stats := mdb.NewClusterStats(fullVersion)
 		stats.SetRedaction(*redaction)
 		stats.SetVerbose(true)
-		if err = CollectCluserDetails(stats, client, connString); err != nil {
-			log.Fatalf(`a valid user with roles 'clusterMonitor' and 'readAnyDatabase' on all mongo processes are required.\n%v`, err)
+		if err = CollectCluserDetails(stats, client, connString, *maobiURL); err != nil {
+			log.Fatalf("a valid user with roles 'clusterMonitor' and 'readAnyDatabase' on all mongo processes are required.\n%v", err)
 		}
 		return
 	} else if *cardinality != "" { // --card <collection> [-v]
@@ -218,11 +232,11 @@ func Run(fullVersion string) {
 			log.Fatal(err)
 		}
 		return
-	} else if *index == true {
+	} else if *index != "" {
 		ix := mdb.NewIndexStats(fullVersion)
 		ix.SetNoColor(*nocolor)
 		ix.SetVerbose(*verbose)
-		if err = CollectIndexStats(ix, client); err != nil {
+		if err = CollectIndexStats(ix, client, *maobiURL); err != nil {
 			log.Fatal(err)
 		}
 		return
@@ -252,7 +266,7 @@ func Run(fullVersion string) {
 	}
 
 	clusterSummary := GetClusterSummary(fullVersion, client)
-	if *info == true {
+	if *info != "" {
 		fmt.Println(clusterSummary)
 		return
 	}

loginfo.go

@@ -2,17 +2,30 @@
 
 package keyhole
 
-import "github.com/simagix/keyhole/mdb"
+import (
+	"fmt"
+
+	"github.com/simagix/keyhole/mdb"
+)
 
 // AnalyzeMongoLogs a helper function to analyze logs
-func AnalyzeMongoLogs(loginfo *mdb.LogInfo, filenames []string) error {
+func AnalyzeMongoLogs(loginfo *mdb.LogInfo, filenames []string, maobiURL string) error {
 	var err error
+	var data []byte
+	var ofile string
 	for _, filename := range filenames {
 		if err = loginfo.AnalyzeFile(filename); err != nil {
-			return err
+			fmt.Println(err)
+			continue
 		}
 		loginfo.Print()
-		loginfo.OutputBSON()
+		if ofile, data, err = loginfo.OutputBSON(); err != nil {
+			fmt.Println(err)
+			continue
+		}
+		if err = GenerateMaobiReport(maobiURL, data, ofile); err != nil {
+			fmt.Println(err)
+		}
 	}
 	return err
 }

maobi.go

@@ -0,0 +1,59 @@
+// Copyright 2020-present Kuei-chun Chen. All rights reserved.
+
+package keyhole
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// GenerateMaobiReport outputs an HTML from Maobi
+func GenerateMaobiReport(maobiURL string, data []byte, ofile string) error {
+	var err error
+	var murl *url.URL
+	if maobiURL == "" {
+		return nil
+	}
+	if murl, err = url.Parse(maobiURL); err != nil {
+		return err
+	}
+	os.Mkdir(htmldir, 0755)
+	i := strings.Index(ofile, ".bson.gz")
+	filename := strings.Replace(ofile[:i]+".html", outdir, htmldir, 1)
+	dial := net.Dialer{Timeout: 2 * time.Second}
+	var c net.Conn
+	to := murl.Hostname()
+	if murl.Port() != "" {
+		to += ":" + murl.Port()
+	}
+	if c, err = dial.Dial("tcp", to); err != nil {
+		return err
+	}
+	c.Close()
+	bodyBuf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(bodyBuf)
+	fileWriter, err := bodyWriter.CreateFormFile("file", ofile)
+	fileWriter.Write(data)
+	contentType := bodyWriter.FormDataContentType()
+	bodyWriter.Close()
+	resp, err := http.Post(maobiURL, contentType, bodyBuf)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	ioutil.WriteFile(filename, body, 0644)
+	fmt.Println(fmt.Sprintf(`HTML report written to %v`, filename))
+	return err
+}

mdb/bson_printer.go

@@ -96,7 +96,6 @@ func (p *BSONPrinter) Print(filename string) error {
 			}
 			cluster.Print()
 		}
-		outdir := "./out/"
 		os.Mkdir(outdir, 0755)
 		ofile := filepath.Base(filename)
 		idx := strings.Index(ofile, ".bson")

mdb/cluster_stats.go

@@ -240,31 +240,31 @@ func (p *ClusterStats) Print() {
 }
 
 // OutputBSON writes bson data to a file
-func (p *ClusterStats) OutputBSON() error {
+func (p *ClusterStats) OutputBSON() (string, []byte, error) {
+	var err error
+	var data []byte
+	var ofile string
 	if p.HostInfo.System.Hostname == "" {
 		result := `Roles 'clusterMonitor' and 'readAnyDatabase' are required`
-		return errors.New(result)
+		return ofile, data, errors.New(result)
 	}
-	var err error
-	var data []byte
 	if data, err = bson.Marshal(p); err != nil {
-		return err
+		return ofile, data, err
 	}
 
-	outdir := "./out"
 	os.Mkdir(outdir, 0755)
 	basename := p.HostInfo.System.Hostname
 	basename = strings.ReplaceAll(basename, ":", "_")
-	ofile := fmt.Sprintf(`%v/%v-stats.bson.gz`, outdir, basename)
+	ofile = fmt.Sprintf(`%v/%v-stats.bson.gz`, outdir, basename)
 	i := 1
 	for DoesFileExist(ofile) {
 		ofile = fmt.Sprintf(`%v/%v.%d-stats.bson.gz`, outdir, basename, i)
 		i++
 	}
 
 	if err = gox.OutputGzipped(data, ofile); err != nil {
-		return err
+		return ofile, data, err
 	}
 	fmt.Println(fmt.Sprintf(`bson data written to %v`, ofile))
-	return err
+	return ofile, data, err
 }

mdb/explain.go

@@ -93,7 +93,6 @@ func (e *Explain) ExecuteAllPlans(client *mongo.Client, filename string) error {
 		if counter == 1 {
 			fmt.Println(stdout)
 		}
-		outdir := "./out/"
 		os.Mkdir(outdir, 0755)
 		ofile := fmt.Sprintf("./out/%v-explain-%03d.json.gz", filepath.Base(filename), counter)
 		data, _ := bson.MarshalExtJSON(document, false, false)