Saving nodes last IPs, countin neighbour subnet nodes

db.go

@@ -34,17 +34,19 @@ func StartNodesKadDataSaver(db *pg.DB, kadDataChan chan *KadDataExt, chunkSize i
 				return merry.Wrap(err)
 			}
 
-			for _, node := range items {
+			for _, nodeI := range items {
+				node := nodeI.(*KadDataExt)
 				var xmax string
 				_, err := tx.QueryOne(&xmax, `
-					INSERT INTO nodes (id, kad_params, location, kad_updated_at, kad_checked_at)
-					VALUES (?, ?, ?, NOW(), NOW())
+					INSERT INTO nodes (id, kad_params, last_ip, location, kad_updated_at, kad_checked_at)
+					VALUES (?, ?, ?, ?, NOW(), NOW())
 					ON CONFLICT (id) DO UPDATE SET
 						kad_params = EXCLUDED.kad_params,
+						last_ip = COALESCE(nodes.last_ip, EXCLUDED.last_ip),
 						location = COALESCE(nodes.location, EXCLUDED.location),
 						kad_updated_at = NOW(),
 						kad_checked_at = GREATEST(nodes.kad_checked_at, EXCLUDED.kad_checked_at)
-					RETURNING xmax`, node.(*KadDataExt).Node.Id, node.(*KadDataExt).Node, node.(*KadDataExt).Location)
+					RETURNING xmax`, node.Node.Id, node.Node, node.IPAddress, node.Location)
 				if err != nil {
 					return merry.Wrap(err)
 				}

geo.go

@@ -22,19 +22,18 @@ func StartLocationSearcher(gdb *geoip.GeoIP, kadDataRawChan chan *pb.Node, kadDa
 	go func() {
 		defer worker.Done()
 		for nodeRaw := range kadDataRawChan {
-			node := &KadDataExt{Node: nodeRaw, Location: nil}
-			ip := nodeRaw.LastIp
-			if ip == "" {
+			node := &KadDataExt{Node: nodeRaw, IPAddress: nodeRaw.LastIp, Location: nil}
+			if node.IPAddress == "" {
 				host, _ := splitToHostAndPort(nodeRaw.Address.Address)
 				ips, err := net.LookupHost(host)
 				if err == nil {
-					ip = ips[0]
+					node.IPAddress = ips[0]
 				} else {
 					logWarn("GEO-LOOKUP", "addr '%s' lookup error: %s", nodeRaw.Address.Address, err)
 				}
 			}
-			if ip != "" {
-				if rec := gdb.GetRecord(ip); rec != nil {
+			if node.IPAddress != "" {
+				if rec := gdb.GetRecord(node.IPAddress); rec != nil {
 					// fmt.Printf("%#v\n", rec)
 					node.Location = &NodeLocation{
 						Country:   rec.CountryName,

migrations/008_nodes_last_ip.go

@@ -0,0 +1,36 @@
+package main
+
+import "github.com/go-pg/migrations/v7"
+
+func init() {
+	migrations.MustRegisterTx(func(db migrations.DB) error {
+		return execSome(db, `
+			ALTER TABLE storjinfo.nodes
+			ADD COLUMN last_ip inet
+			`, `
+			UPDATE storjinfo.nodes
+			SET last_ip = (kad_params->>'last_ip')::inet
+			WHERE kad_params ? 'last_ip'
+			`, `
+			CREATE FUNCTION node_last_ip_subnet(ip inet)
+				RETURNS cidr
+			AS
+			$BODY$
+				SELECT set_masklen(ip::cidr, 24);
+			$BODY$
+			LANGUAGE sql
+			IMMUTABLE
+			`, `
+			CREATE INDEX nodes__last_ip_subnet__index ON nodes (node_last_ip_subnet(last_ip))
+			`)
+	}, func(db migrations.DB) error {
+		return execSome(db, `
+			DROP INDEX nodes__last_ip_subnet__index
+			`, `
+			DROP FUNCTION node_last_ip_subnet
+			`, `
+			ALTER TABLE storjinfo.nodes
+			DROP COLUMN last_ip
+			`)
+	})
+}

server.go

@@ -391,7 +391,9 @@ func HandleNode(wr http.ResponseWriter, r *http.Request, ps httprouter.Params) e
 	}
 
 	node := &Node{ID: nodeID}
-	err = db.Model(node).Column("created_at", "kad_params", "kad_updated_at", "self_updated_at", "self_params", "location").Where("id = ?", nodeID).Select()
+	err = db.Model(node).
+		Column("created_at", "kad_params", "kad_updated_at", "self_updated_at", "self_params", "last_ip", "location").
+		WherePK().Select()
 	if err == pg.ErrNoRows {
 		appendFileString("unknown_nodes_log.txt", time.Now().Format("2006-01-02 15:04:05")+" "+node.ID.String()+"\n")
 		return render(wr, r, http.StatusBadRequest, "node.html", "base", map[string]interface{}{
@@ -404,6 +406,19 @@ func HandleNode(wr http.ResponseWriter, r *http.Request, ps httprouter.Params) e
 		return merry.Wrap(err)
 	}
 
+	subnetNeighborsCount := -1
+	if node.LastIP != nil {
+		_, err = db.QueryOne(&subnetNeighborsCount, `
+			SELECT count(*) FROM nodes
+			WHERE node_last_ip_subnet(last_ip) = node_last_ip_subnet(?)
+			  AND last_ip != ?
+			  AND self_updated_at > NOW() - INTERVAL '24 hours'
+			`, node.LastIP, node.LastIP)
+		if err != nil {
+			return merry.Wrap(err)
+		}
+	}
+
 	query := r.URL.Query()
 	statsTimeFrom := extractMonthStartTimeUTC(query)
 	var dailyHistories []*NodeHistory
@@ -419,14 +434,15 @@ func HandleNode(wr http.ResponseWriter, r *http.Request, ps httprouter.Params) e
 	statsNextMonthTime := statsTimeFrom.AddDate(0, 1, 0)
 
 	return render(wr, r, http.StatusOK, "node.html", "base", map[string]interface{}{
-		"NodeIDStr":          node.ID.String(),
-		"Node":               node,
-		"NodeHistory":        GroupNodeHistories(dailyHistories),
-		"NodeType_STORAGE":   pb.NodeType_STORAGE,
-		"StatsTimeFrom":      statsTimeFrom,
-		"StatsPrevMonthTime": statsPrevMonthTime,
-		"StatsNextMonthTime": statsNextMonthTime,
-		"Query":              QueryExt{query},
+		"NodeIDStr":            node.ID.String(),
+		"Node":                 node,
+		"NodeHistory":          GroupNodeHistories(dailyHistories),
+		"NodeType_STORAGE":     pb.NodeType_STORAGE,
+		"StatsTimeFrom":        statsTimeFrom,
+		"StatsPrevMonthTime":   statsPrevMonthTime,
+		"StatsNextMonthTime":   statsNextMonthTime,
+		"SubnetNeighborsCount": subnetNeighborsCount,
+		"Query":                QueryExt{query},
 	})
 }
 

utils.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"math"
+	"net"
 	"os"
 	"strconv"
 	"strings"
@@ -198,8 +199,9 @@ func (l *NodeLocation) Value() (driver.Value, error) {
 }
 
 type KadDataExt struct {
-	Node     *pb.Node
-	Location *NodeLocation `sql:"composite:node_Location"`
+	Node      *pb.Node
+	IPAddress string
+	Location  *NodeLocation `sql:"composite:node_Location"`
 }
 
 func scanCompositePairStr(val interface{}) (string, string, string, error) {
@@ -446,6 +448,7 @@ type Node struct {
 	SelfParams    *pb.NodeInfoResponse
 	KadUpdatedAt  time.Time
 	SelfUpdatedAt time.Time
+	LastIP        net.IP
 	Location      *NodeLocation `sql:"composite:node_Location"`
 }
 

www/css/main.css

@@ -10,9 +10,16 @@
 	font-size: 85%;
 }
 
+.ok {
+	background-color: #cfc;
+}
 .warn {
 	background-color: #ff9;
 }
+.ok, .warn {
+	padding: 1px;
+	border-radius: 2px;
+}
 
 .wide-block {
 	overflow-x: auto;

www/templates/node.html

@@ -46,6 +46,26 @@ <h1 class="node-id">
 
 {{template "node_id_info" .L.With .Node.ID}}
 
+<p>
+	<b>IP:</b>
+	{{if .Node.LastIP}}
+	{{.Node.LastIP}},
+	{{if eq .SubnetNeighborsCount 0}}
+	<span class="ok">{{.L.Loc "the only node in /24-subnet" "ru" "единственная нода в /24-подсети"}}</span>
+	{{else}}
+	<span class="warn">
+		{{if .L.Is "ru"}}
+		ещё {{.SubnetNeighborsCount}} {{.L.Pluralize .SubnetNeighborsCount "нода" "ноды" "нод"}} находятся в той же /24-подсети
+		{{else}}
+		{{.SubnetNeighborsCount}} more {{.L.Pluralize .SubnetNeighborsCount "node" "nodes"}} are in the same /24-subnet
+		{{end}}
+	</span>
+	{{end}}
+	{{else}}
+	{{.L.Loc "unknown yet" "ru" "пока неизвестен"}}
+	{{end}}
+</p>
+
 <h2>{{.L.Loc "Statistics" "ru" "Статистика"}}</h2>
 
 <p>