Filter VRPs if they have expired. Prevent stale JSON files from linge…

…ring

First, VRPs that have expiry times are now checked, and they are
filtered out at import time.

Second, If a VRP JSON file is too old, and the "current state"
(in the case of a update) is too old, the state will empty to avoid
routing on old data.

Third, Every time a refresh cycle now happens, the file is reprocessed
to check for expiry, if the resulting VRPs from that procesing changes
then a new update+serial is pushed

Tag: #15

cmd/stayrtr/stayrtr.go

@@ -200,6 +200,12 @@ func isValidPrefixLength(prefix *net.IPNet, maxLength uint8) bool {
 func processData(vrplistjson []prefixfile.VRPJson) ([]rtr.VRP, int, int, int) {
 	filterDuplicates := make(map[string]bool)
 
+	// It may be tempting to change this to a simple time.Since() but that will
+	// grab the current time every time it's invoked, time calls can be slow on
+	// some platforms, so lets just get the unix time when we start and use that
+	// to compare it all
+	NowUnix := time.Now().UTC().Unix()
+
 	var vrplist []rtr.VRP
 	var countv4 int
 	var countv6 int
@@ -220,6 +226,14 @@ func processData(vrplistjson []prefixfile.VRPJson) ([]rtr.VRP, int, int, int) {
 			continue
 		}
 
+		if v.Expires != nil {
+			// Prevent stale VRPs from being considered
+			// https://github.com/bgp/stayrtr/issues/15
+			if int(NowUnix) > int(*v.Expires) {
+				continue
+			}
+		}
+
 		if prefix.IP.To4() != nil {
 			countv4++
 		} else {
@@ -251,6 +265,8 @@ func (e IdenticalFile) Error() string {
 	return fmt.Sprintf("File %s is identical to the previous version", e.File)
 }
 
+var errVRPJsonFileTooOld = errors.New("VRP JSON file is older than 24 hours")
+
 // Update the state based on the current slurm file and data.
 func (s *state) updateFromNewState() error {
 	sessid := s.server.GetSessionId()
@@ -260,14 +276,15 @@ func (s *state) updateFromNewState() error {
 		return nil
 	}
 
+	buildtime, err := time.Parse(time.RFC3339, s.lastdata.Metadata.Buildtime)
 	if s.checktime {
-		buildtime, err := time.Parse(time.RFC3339, s.lastdata.Metadata.Buildtime)
 		if err != nil {
 			return err
 		}
 		notafter := buildtime.Add(time.Hour * 24)
 		if time.Now().UTC().After(notafter) {
-			return errors.New(fmt.Sprintf("VRP JSON file is older than 24 hours: %v", buildtime))
+			log.Warnf("VRP JSON file is older than 24 hours: %v", buildtime)
+			return errVRPJsonFileTooOld
 		}
 	}
 
@@ -281,7 +298,46 @@ func (s *state) updateFromNewState() error {
 	vrps, count, countv4, countv6 := processData(vrpsjson)
 
 	log.Infof("New update (%v uniques, %v total prefixes).", len(vrps), count)
+	return s.applyUpdateFromNewState(vrps, sessid, vrpsjson, countv4, countv6)
+}
+
+// Update the state based on the currently loaded files
+func (s *state) reloadFromCurrentState() error {
+	sessid := s.server.GetSessionId()
+
+	vrpsjson := s.lastdata.Data
+	if vrpsjson == nil {
+		return nil
+	}
 
+	buildtime, err := time.Parse(time.RFC3339, s.lastdata.Metadata.Buildtime)
+	if s.checktime {
+		if err != nil {
+			return err
+		}
+		notafter := buildtime.Add(time.Hour * 24)
+		if time.Now().UTC().After(notafter) {
+			log.Warnf("VRP JSON file is older than 24 hours: %v", buildtime)
+			return errVRPJsonFileTooOld
+		}
+	}
+
+	if s.slurm != nil {
+		kept, removed := s.slurm.FilterOnVRPs(vrpsjson)
+		asserted := s.slurm.AssertVRPs()
+		log.Infof("Slurm filtering: %v kept, %v removed, %v asserted", len(kept), len(removed), len(asserted))
+		vrpsjson = append(kept, asserted...)
+	}
+
+	vrps, count, countv4, countv6 := processData(vrpsjson)
+	if s.server.CountVRPs() != count {
+		log.Infof("New update to old state (%v uniques, %v total prefixes). (old %v - new %v)", len(vrps), count, s.server.CountVRPs(), count)
+		return s.applyUpdateFromNewState(vrps, sessid, vrpsjson, countv4, countv6)
+	}
+	return nil
+}
+
+func (s *state) applyUpdateFromNewState(vrps []rtr.VRP, sessid uint16, vrpsjson []prefixfile.VRPJson, countv4 int, countv6 int) error {
 	s.server.AddVRPs(vrps)
 
 	serial, _ := s.server.GetCurrentSerial(sessid)
@@ -430,8 +486,21 @@ func (s *state) routineUpdate(file string, interval int, slurmFile string) {
 		if cacheUpdated || slurmNotPresentOrUpdated {
 			err := s.updateFromNewState()
 			if err != nil {
+				if err == errVRPJsonFileTooOld {
+					// If the exiting build time is over 24 hours, It's time to drop everything out.
+					// to avoid routing on stale data
+					buildTime := s.exported.Metadata.GetBuildTime()
+					if !buildTime.IsZero() && time.Since(buildTime) > time.Hour*24 {
+						s.server.AddVRPs([]rtr.VRP{}) // empty the store of VRP by giving it a empty VRP array, triggering a emptying
+					}
+				}
 				log.Errorf("Error updating from new state: %v", err)
 			}
+		} else {
+			err := s.reloadFromCurrentState()
+			if err != nil {
+				log.Errorf("Error updating state: %v", err)
+			}
 		}
 	}
 }

cmd/stayrtr/stayrtr_test.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"os"
 	"testing"
+	"time"
 
 	rtr "github.com/bgp/stayrtr/lib"
 	"github.com/bgp/stayrtr/prefixfile"
@@ -13,12 +14,16 @@ import (
 
 func TestProcessData(t *testing.T) {
 	var stuff []prefixfile.VRPJson
+	NowUnix := int(time.Now().Unix())
+	ExpiredTime := 1337
+
 	stuff = append(stuff,
 		prefixfile.VRPJson{
-			Prefix: "192.168.0.0/24",
-			Length: 24,
-			ASN:    123,
-			TA:     "testrir",
+			Prefix:  "192.168.0.0/24",
+			Length:  24,
+			ASN:     123,
+			TA:      "testrir",
+			Expires: &NowUnix,
 		},
 		prefixfile.VRPJson{
 			Prefix: "192.168.0.0/24",
@@ -86,6 +91,14 @@ func TestProcessData(t *testing.T) {
 			ASN:    "ASN123",
 			TA:     "testrir",
 		},
+		// Invalid. Has expired
+		prefixfile.VRPJson{
+			Prefix:  "192.168.2.0/24",
+			Length:  24,
+			ASN:     124,
+			TA:      "testrir",
+			Expires: &ExpiredTime,
+		},
 	)
 	got, count, v4count, v6count := processData(stuff)
 	want := []rtr.VRP{
@@ -145,6 +158,9 @@ func TestJson(t *testing.T) {
 		t.Errorf("Unable to decode json: %v", err)
 	}
 
+	Ex1 := 1627568318
+	Ex2 := 1627575699
+
 	want := (&prefixfile.VRPList{
 		Metadata: prefixfile.MetaData{
 			Counts:    2,
@@ -155,14 +171,14 @@ func TestJson(t *testing.T) {
 				Length:  24,
 				ASN:     float64(13335),
 				TA:      "apnic",
-				Expires: 1627568318,
+				Expires: &Ex1,
 			},
 			{
 				Prefix:  "2001:200:136::/48",
 				Length:  48,
 				ASN:     "AS9367",
 				TA:      "apnic",
-				Expires: 1627575699,
+				Expires: &Ex2,
 			},
 		},
 	})

lib/server.go

@@ -344,6 +344,13 @@ func (s *Server) SetSerial(serial uint32) {
 	s.setSerial(serial)
 }
 
+func (s *Server) CountVRPs() int {
+	s.vrplock.RLock()
+	defer s.vrplock.RUnlock()
+
+	return len(s.vrpCurrent)
+}
+
 func (s *Server) AddVRPs(vrps []VRP) {
 	s.vrplock.RLock()
 

prefixfile/prefixfile.go

@@ -5,21 +5,30 @@ import (
 	"net"
 	"strconv"
 	"strings"
+	"time"
 )
 
 type VRPJson struct {
 	Prefix  string      `json:"prefix"`
 	Length  uint8       `json:"maxLength"`
 	ASN     interface{} `json:"asn"`
 	TA      string      `json:"ta,omitempty"`
-	Expires int         `json:"expires,omitempty"`
+	Expires *int        `json:"expires,omitempty"`
 }
 
 type MetaData struct {
 	Counts    int    `json:"vrps"`
 	Buildtime string `json:"buildtime,omitempty"`
 }
 
+func (md MetaData) GetBuildTime() time.Time {
+	bt, err := time.Parse(time.RFC3339, md.Buildtime)
+	if err != nil {
+		return time.Time{}
+	}
+	return bt
+}
+
 type VRPList struct {
 	Metadata MetaData  `json:"metadata,omitempty"`
 	Data     []VRPJson `json:"roas"` // for historical reasons this is called 'roas', but should've been called vrps