add some stats (nb of imported nodes + execution time)

`-no-dro` option is deprecated for a `-drop` option (it's reverse)
claranet · Dec 3, 2019 · 9544162 · 9544162
1 parent bece84c
commit 9544162
Show file tree

Hide file tree

Showing 29 changed files with 55 additions and 17 deletions.
diff --git a/addresses.go b/addresses.go
@@ -63,6 +63,7 @@ func (list addressList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]addressNodes)
 	n := make(map[string]string)

diff --git a/autoscalinggroups.go b/autoscalinggroups.go
@@ -84,6 +84,7 @@ func (list autoScalingGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]autoScalingGroupNodes)
 	n := make(map[string]string)

diff --git a/availibiltyzones.go b/availibiltyzones.go
@@ -54,6 +54,7 @@ func (list availabilityZoneList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]availabilityZoneNodes)
 	n := make(map[string]string)

diff --git a/cacheclusters.go b/cacheclusters.go
@@ -113,4 +113,5 @@ func (list cacheClusterList) addEdges(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 }
diff --git a/cachesubnetgroups.go b/cachesubnetgroups.go
@@ -59,6 +59,7 @@ func (list cacheSubnetGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]cacheSubnetGroupNodes)
 	n := make(map[string]string)

diff --git a/connector.go b/connector.go
@@ -24,6 +24,11 @@ type connector struct {
 	awsAccountID   string
 	ressources     map[string]map[string]string
 	waitGroup      sync.WaitGroup
+	stats          statistics
+}
+
+type statistics struct {
+	NumberOfNodes int
 }
 
 func newConnector(profile, region, dgraph *string) *connector {

diff --git a/dbclusterparametergroups.go b/dbclusterparametergroups.go
@@ -61,6 +61,7 @@ func (list dbClusterParameterGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]dbClusterParameterGroupNodes)
 	n := make(map[string]string)

diff --git a/dbclusters.go b/dbclusters.go
@@ -101,6 +101,7 @@ func (list dbClusterList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]dbClusterNodes)
 	n := make(map[string]string)

diff --git a/dbinstances.go b/dbinstances.go
@@ -115,6 +115,7 @@ func (list dbInstanceList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]dbInstanceNodes)
 	n := make(map[string]string)

diff --git a/dbparametergroups.go b/dbparametergroups.go
@@ -61,6 +61,7 @@ func (list dbParameterGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]dbParameterGroupNodes)
 	n := make(map[string]string)

diff --git a/dbsubnetgroups.go b/dbsubnetgroups.go
@@ -63,6 +63,7 @@ func (list dbSubnetGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]dbSubnetGroupNodes)
 	n := make(map[string]string)

diff --git a/dgraph.go b/dgraph.go
@@ -12,7 +12,7 @@ import (
 )
 
 func (c *connector) dgraphDropAll() {
-	log.Println("Drop all previous data")
+	log.Println("Drop all data")
 	op := api.Operation{DropAll: true}
 	if err := c.dgraphClient.Alter(*c.context, &op); err != nil {
 		log.Fatal(err)
@@ -25,9 +25,9 @@ func (c *connector) dgraphDropPrevious() {
 	defer txn.Discard(*c.context)
 
 	q := `query query($owner: string, $region: string){
-		list(func: eq(OwnerId, $owner)) @filter(eq(Region, $region)) {
-			uid
-		}
+			list(func: eq(OwnerId, $owner)) @filter(eq(Region, $region)) {
+				uid
+			}
 		}`
 
 	res, err := txn.QueryWithVars(*c.context, q, map[string]string{"$owner": c.awsAccountID, "$region": c.awsRegion})

diff --git a/images.go b/images.go
@@ -93,6 +93,7 @@ func (list imageList) addNodes(c *connector) {
 	}
 
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]imageNodes)
 	n := make(map[string]string)

diff --git a/instanceprofiles.go b/instanceprofiles.go
@@ -60,6 +60,7 @@ func (list instanceProfileList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]instanceProfileNodes)
 	n := make(map[string]string)

diff --git a/instances.go b/instances.go
@@ -53,7 +53,7 @@ func (list instanceList) addNodes(c *connector) {
 		return
 	}
 	log.Println("Add Instance Nodes")
-	instances := make(instanceNodes, 0, len(list.Reservations))
+	a := make(instanceNodes, 0, len(list.Reservations))
 
 	for _, r := range list.Reservations {
 		for _, i := range r.Instances {
@@ -81,10 +81,11 @@ func (list instanceList) addNodes(c *connector) {
 			if i.PublicIpAddress != nil {
 				b.PublicIPAddress = *i.PublicIpAddress
 			}
-			instances = append(instances, b)
+			a = append(a, b)
 		}
 	}
-	c.dgraphAddNodes(instances)
+	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]instanceNodes)
 	n := make(map[string]string)

diff --git a/keypairs.go b/keypairs.go
@@ -55,6 +55,7 @@ func (list keyPairList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]keyPairNodes)
 	n := make(map[string]string)

diff --git a/launchconfigurations.go b/launchconfigurations.go
@@ -71,6 +71,7 @@ func (list launchConfigurationList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]launchConfigurationNodes)
 	n := make(map[string]string)

diff --git a/launchtemplates.go b/launchtemplates.go
@@ -63,6 +63,7 @@ func (list launchTemplateList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]launchTemplateNodes)
 	n := make(map[string]string)

diff --git a/loadbalancers.go b/loadbalancers.go
@@ -93,6 +93,7 @@ func (list loadBalancerList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]loadBalancerNodes)
 	n := make(map[string]string)
@@ -131,6 +132,7 @@ func (list loadBalancerV2List) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]loadBalancerNodes)
 	n := make(map[string]string)

diff --git a/main.go b/main.go
@@ -3,7 +3,9 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log"
 	"os"
+	"time"
 )
 
 func main() {
@@ -12,33 +14,34 @@ func main() {
 	dgraph := flag.String("dgraph", "127.0.0.1:9080", "Dgraph server (ip:port)")
 	dtype := flag.String("type", "", "Get the schema for a type (only after importing some data)")
 	list := flag.Bool("list", false, "List available ressource types")
-	nodrop := flag.Bool("no-drop", false, "Disable the drop of all nodes and the schema before import")
+	drop := flag.Bool("drop", false, "Drop all nodes and the schema")
 	noschema := flag.Bool("no-schema", false, "Disable the refresh schema at each run")
 	flag.Parse()
 
-	if *dtype != "" {
-		dgraphDisplaySchema(dgraph, dtype)
+	if *list == true {
+		fmt.Println("Address, AutoScalingGroup, AvailabilityZone, CacheCluster, CacheSubnetGroup, Cidr, DbCluster, DbClusterParameterGroup, DbInstance, DbParameterGroup, DbSubnetGroup, Instance, InstanceProfile, KeyPair, LaunchConfiguration, LaunchTemplate, LoadBalancer, NatGateway, OptionGroup, SecurityGroup, Subnet, TargetGroup, Volume, Vpc, VpcPeeringConnection")
 		os.Exit(0)
 	}
 
 	connector := newConnector(profile, region, dgraph)
 	defer connector.grpcConnexion.Close()
 
-	if *list == true {
-		fmt.Println("Address, AutoScalingGroup, AvailabilityZone, CacheCluster, CacheSubnetGroup, Cidr, DbCluster, DbClusterParameterGroup, DbInstance, DbParameterGroup, DbSubnetGroup, Instance, InstanceProfile, KeyPair, LaunchConfiguration, LaunchTemplate, LoadBalancer, NatGateway, OptionGroup, SecurityGroup, Subnet, TargetGroup, Volume, Vpc, VpcPeeringConnection")
+	if *drop == true {
+		connector.dgraphDropAll()
 		os.Exit(0)
 	}
 
-	if *nodrop != true {
-		connector.dgraphDropAll()
-	} else {
-		connector.dgraphDropPrevious()
+	if *dtype != "" {
+		dgraphDisplaySchema(dgraph, dtype)
+		os.Exit(0)
 	}
 
 	if *noschema != true {
 		connector.dgraphAddSchema()
 	}
 
+	connector.dgraphDropPrevious()
+
 	var instances instanceList
 	var keypairs keyPairList
 	var volumes volumeList
@@ -69,6 +72,7 @@ func main() {
 
 	// List ressources
 	connector.waitGroup.Add(25)
+	start := time.Now()
 	go func() { instances = connector.listInstances() }()
 	go func() { keypairs = connector.listKeyPairs() }()
 	go func() { volumes = connector.listVolumes() }()
@@ -153,4 +157,6 @@ func main() {
 	// snapshots.addEdges(connector)
 
 	connector.waitGroup.Wait()
+
+	log.Printf("%v Nodes have been imported in %s\n", connector.stats.NumberOfNodes, time.Since(start))
 }
diff --git a/natgateways.go b/natgateways.go
@@ -68,6 +68,7 @@ func (list natGatewayList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]natGatewayNodes)
 	n := make(map[string]string)

diff --git a/optionsgroups.go b/optionsgroups.go
@@ -65,6 +65,7 @@ func (list optionGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]optionGroupNodes)
 	n := make(map[string]string)

diff --git a/securitygroups.go b/securitygroups.go
@@ -108,6 +108,7 @@ func (list securityGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	if len(y) != 0 {
 		for i := range y {
@@ -118,6 +119,7 @@ func (list securityGroupList) addNodes(c *connector) {
 			})
 		}
 		c.dgraphAddNodes(z)
+		c.stats.NumberOfNodes += len(z)
 	}
 
 	m := make(map[string]securityGroupNodes)

diff --git a/snapshots.go b/snapshots.go
@@ -74,8 +74,8 @@ func (list snapshotList) addNodes(c *connector) {
 		b.Progress = *i.Progress
 		a = append(a, b)
 	}
-
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]snapshotNodes)
 	n := make(map[string]string)

diff --git a/subnets.go b/subnets.go
@@ -74,6 +74,7 @@ func (list subnetList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]subnetNodes)
 	n := make(map[string]string)

diff --git a/targetgroups.go b/targetgroups.go
@@ -77,6 +77,7 @@ func (list targetGroupList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]targetGroupNodes)
 	n := make(map[string]string)

diff --git a/volumes.go b/volumes.go
@@ -76,6 +76,7 @@ func (list volumeList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]volumeNodes)
 	n := make(map[string]string)

diff --git a/vpcpeeringconnections.go b/vpcpeeringconnections.go
@@ -64,6 +64,7 @@ func (list vpcPeeringConnectionList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]vpcPeeringConnectionNodes)
 	n := make(map[string]string)

diff --git a/vpcs.go b/vpcs.go
@@ -61,6 +61,7 @@ func (list vpcList) addNodes(c *connector) {
 		a = append(a, b)
 	}
 	c.dgraphAddNodes(a)
+	c.stats.NumberOfNodes += len(a)
 
 	m := make(map[string]vpcNodes)
 	n := make(map[string]string)