[TASK] improve output (filtering, fix links type, new outputs nodelis…

…t for freifunkapi and meshviewer-ffrgb v10 NextGen)

INSTALL.md

@@ -59,7 +59,7 @@ accessible under `/var/www/html/meshviewer`.
 #### With webserver (Apache, nginx)
 The meshviewer needs the output files like `nodes_path` and `graph_path` inside
 the same directory as the `dataPath`. Change the path in the section
-`[meshviewer]` accordingly.
+`[[nodes.output.meshviewer]]` accordingly.
 
 ### Service
 ```bash

cmd/serve.go

@@ -8,8 +8,9 @@ import (
 	"time"
 
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/database/all"
-	"github.com/FreifunkBremen/yanic/meshviewer"
+	allDatabase "github.com/FreifunkBremen/yanic/database/all"
+	"github.com/FreifunkBremen/yanic/output"
+	allOutput "github.com/FreifunkBremen/yanic/output/all"
 	"github.com/FreifunkBremen/yanic/respond"
 	"github.com/FreifunkBremen/yanic/runtime"
 	"github.com/FreifunkBremen/yanic/webserver"
@@ -24,7 +25,7 @@ var serveCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		config := loadConfig()
 
-		connections, err := all.Connect(config.Database.Connection)
+		connections, err := allDatabase.Connect(config.Database.Connection)
 		if err != nil {
 			panic(err)
 		}
@@ -33,7 +34,13 @@ var serveCmd = &cobra.Command{
 
 		nodes = runtime.NewNodes(config)
 		nodes.Start()
-		meshviewer.Start(config, nodes)
+
+		outputs, err := allOutput.Register(config.Nodes.Output)
+		if err != nil {
+			panic(err)
+		}
+		output.Start(outputs, nodes, config)
+		defer output.Close()
 
 		if config.Webserver.Enable {
 			log.Println("starting webserver on", config.Webserver.Bind)

config_example.toml

@@ -34,7 +34,7 @@ save_interval = "5s"
 offline_after = "10m"
 
 
-[meshviewer]
+[[nodes.output.meshviewer]]
 # The structure version of the output which should be generated (i.e. nodes.json)
 # version 1 is accepted by the legacy meshviewer (which is the master branch)
 #   i.e. https://github.com/ffnord/meshviewer/tree/master
@@ -48,6 +48,26 @@ nodes_path = "/var/www/html/meshviewer/data/nodes.json"
 graph_path = "/var/www/html/meshviewer/data/graph.json"
 
 
+[nodes.output.meshviewer.filter]
+# no_owner = true
+has_location = true
+blacklist = ["vpnid"]
+
+[nodes.output.meshviewer.filter.in_area]
+latitude_min = 34.30
+latitude_max = 71.85
+longitude_min = -24.96
+longitude_max = 39.72
+
+[[nodes.output.nodelist]]
+enable   = true
+path = "/var/www/html/meshviewer/data/nodelist.json"
+
+[[nodes.output.meshviewer-ffrgb]]
+enable   = true
+path = "/var/www/html/meshviewer/data/meshviewer.json"
+
+
 [database]
 # this will send delete commands to the database to prune data
 # which is older than:

data/nodeinfo.go

@@ -4,7 +4,7 @@ package data
 type NodeInfo struct {
 	NodeID   string    `json:"node_id"`
 	Network  Network   `json:"network"`
-	Owner    *Owner    `json:"-"` // Removed for privacy reasons
+	Owner    *Owner    `json:"owner"`
 	System   System    `json:"system"`
 	Hostname string    `json:"hostname"`
 	Location *Location `json:"location,omitempty"`

database/influxdb/global_test.go

@@ -22,37 +22,43 @@ func TestGlobalStats(t *testing.T) {
 func createTestNodes() *runtime.Nodes {
 	nodes := runtime.NewNodes(&runtime.Config{})
 
-	nodeData := &data.ResponseData{
+	nodeData := &runtime.Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 23,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "abcdef012345",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	}
-	nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
-	nodes.Update("abcdef012345", nodeData)
+	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodes.AddNode(nodeData)
 
-	nodes.Update("112233445566", &data.ResponseData{
+	nodes.AddNode(&runtime.Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 2,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "112233445566",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	})
 
-	nodes.Update("0xdeadbeef0x", &data.ResponseData{
-		NodeInfo: &data.NodeInfo{
-			VPN: true,
+	nodes.AddNode(&runtime.Node{
+		Online: true,
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},

database/influxdb/node_test.go

@@ -144,7 +144,7 @@ func testPoints(nodes ...*runtime.Node) (points []*client.Point) {
 	}
 
 	for _, node := range nodes {
-		nodesList.Update(node.Nodeinfo.NodeID, &data.ResponseData{NodeInfo: node.Nodeinfo})
+		nodesList.AddNode(node)
 	}
 
 	// Process data

output/all/filter.go

@@ -0,0 +1,39 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+// Config Filter
+type filterConfig map[string]interface{}
+
+type filterFunc func(*runtime.Node) *runtime.Node
+
+func noFilter(node *runtime.Node) *runtime.Node {
+	return node
+}
+
+// Create Filter
+func (f filterConfig) filtering(nodesOrigin *runtime.Nodes) *runtime.Nodes {
+	nodes := runtime.NewNodes(&runtime.Config{})
+	filterfuncs := []filterFunc{
+		f.HasLocation(),
+		f.Blacklist(),
+		f.InArea(),
+		f.NoOwner(),
+	}
+
+	for _, nodeOrigin := range nodesOrigin.List {
+		//maybe cloning of this object is better?
+		node := nodeOrigin
+		for _, f := range filterfuncs {
+			node = f(node)
+			if node == nil {
+				break
+			}
+		}
+
+		if node != nil {
+			nodes.AddNode(node)
+		}
+	}
+	return nodes
+}

output/all/filter_blacklist.go

@@ -0,0 +1,24 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+func (f filterConfig) Blacklist() filterFunc {
+	v, ok := f["blacklist"]
+	if !ok {
+		return noFilter
+	}
+
+	list := make(map[string]interface{})
+	for _, nodeid := range v.([]interface{}) {
+		list[nodeid.(string)] = true
+	}
+
+	return func(node *runtime.Node) *runtime.Node {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if _, ok := list[nodeinfo.NodeID]; ok {
+				return nil
+			}
+		}
+		return node
+	}
+}

output/all/filter_blacklist_test.go

@@ -0,0 +1,34 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterBlacklist(t *testing.T) {
+	assert := assert.New(t)
+	var config filterConfig
+
+	config = map[string]interface{}{}
+
+	filterBlacklist := config.Blacklist()
+
+	n := filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	config["blacklist"] = []interface{}{"a", "c"}
+	filterBlacklist = config.Blacklist()
+
+	n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{NodeID: "a"}})
+	assert.Nil(n)
+
+	n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	n = filterBlacklist(&runtime.Node{})
+	assert.NotNil(n)
+
+}

output/all/filter_haslocation.go

@@ -0,0 +1,26 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+func (f filterConfig) HasLocation() filterFunc {
+	withLocation, ok := f["has_location"].(bool)
+	if !ok {
+		return noFilter
+	}
+	return func(node *runtime.Node) *runtime.Node {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if withLocation {
+				if location := nodeinfo.Location; location != nil {
+					return node
+				}
+			} else {
+				if location := nodeinfo.Location; location == nil {
+					return node
+				}
+			}
+		} else if !withLocation {
+			return node
+		}
+		return nil
+	}
+}

output/all/filter_haslocation_test.go

@@ -0,0 +1,50 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterHasLocation(t *testing.T) {
+	assert := assert.New(t)
+	var config filterConfig
+
+	config = map[string]interface{}{}
+
+	filterHasLocation := config.HasLocation()
+	n := filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.NotNil(n)
+
+	config["has_location"] = true
+	filterHasLocation = config.HasLocation()
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.NotNil(n)
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.Nil(n)
+
+	n = filterHasLocation(&runtime.Node{})
+	assert.Nil(n)
+
+	config["has_location"] = false
+	filterHasLocation = config.HasLocation()
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.Nil(n)
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	n = filterHasLocation(&runtime.Node{})
+	assert.NotNil(n)
+}