Merge pull request #782 from NebulousLabs/hostdb

Hostdb

Makefile

@@ -62,11 +62,10 @@ clean:
 # command. 'test' runs short tests that should last no more than a few seconds,
 # 'test-long' runs more thorough tests which should not last more than a few
 # minutes.
-pkgs = ./api ./compatibility ./crypto ./encoding ./modules/consensus \
-       ./modules/gateway ./modules/host ./modules/hostdb		     \
-       ./modules/miner ./modules/renter ./modules/transactionpool    \
-       ./modules/wallet ./modules/explorer ./persist                 \
-       ./siae ./types
+pkgs = ./api ./compatibility ./crypto ./encoding ./modules ./modules/consensus \
+       ./modules/explorer ./modules/gateway ./modules/host ./modules/hostdb \
+       ./modules/miner ./modules/renter ./modules/transactionpool \
+       ./modules/wallet ./persist ./siae ./types
 test: clean fmt REBUILD
 	go test -short -tags='debug testing' -timeout=10s $(pkgs)
 test-v: clean fmt REBUILD

modules/gateway.go

@@ -32,21 +32,6 @@ type PeerConn interface {
 // keeping the connection open after all necessary I/O has been performed.
 type RPCFunc func(PeerConn) error
 
-// A NetAddress contains the information needed to contact a peer.
-type NetAddress string
-
-// Host returns the NetAddress' IP.
-func (na NetAddress) Host() string {
-	host, _, _ := net.SplitHostPort(string(na))
-	return host
-}
-
-// Port returns the NetAddress' port number.
-func (na NetAddress) Port() string {
-	_, port, _ := net.SplitHostPort(string(na))
-	return port
-}
-
 // A Gateway facilitates the interactions between the local node and remote
 // nodes (peers). It relays incoming blocks and transactions to local modules,
 // and broadcasts outgoing blocks and transactions to peers. In a broad sense,

modules/hostdb/hostentry.go

@@ -1,8 +1,7 @@
 package hostdb
 
 import (
-	"net"
-
+	"github.com/NebulousLabs/Sia/build"
 	"github.com/NebulousLabs/Sia/modules"
 	"github.com/NebulousLabs/Sia/types"
 )
@@ -17,11 +16,14 @@ type hostEntry struct {
 // insert adds a host entry to the state. The host will be inserted into the
 // set of all hosts, and if it is online and responding to requests it will be
 // put into the list of active hosts.
+//
+// TODO: Function should return an error.
 func (hdb *HostDB) insertHost(host modules.HostSettings) {
-	// Blacklist localhost.
-	if host.IPAddress.Host() == "localhost" {
+	// Remove garbage hosts and local hosts.
+	if !host.IPAddress.IsValid() {
 		return
-	} else if ip := net.ParseIP(string(host.IPAddress)); ip != nil && ip.IsLoopback() {
+	}
+	if host.IPAddress.IsLocal() && build.Release != "testing" {
 		return
 	}
 

modules/netaddress.go

@@ -0,0 +1,52 @@
+package modules
+
+import (
+	"net"
+)
+
+// A NetAddress contains the information needed to contact a peer.
+type NetAddress string
+
+// Host removes the port from a NetAddress, returning just the host. If the
+// address is invalid, the empty string is returned.
+func (na NetAddress) Host() string {
+	host, _, err := net.SplitHostPort(string(na))
+	// 'host' is not always the empty string if an error is returned.
+	if err != nil {
+		return ""
+	}
+	return host
+}
+
+// Port returns the NetAddress object's port number. The empty string is
+// returned if the NetAddress is invalid.
+func (na NetAddress) Port() string {
+	_, port, err := net.SplitHostPort(string(na))
+	// 'port' will not always be the empty string if an error is returned.
+	if err != nil {
+		return ""
+	}
+	return port
+}
+
+// IsLocal returns true for ip addresses that are on the same machine.
+func (na NetAddress) IsLocal() bool {
+	if !na.IsValid() {
+		return false
+	}
+
+	host := na.Host()
+	if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
+		return true
+	}
+	if host == "localhost" {
+		return true
+	}
+	return false
+}
+
+// IsValid does nothing. Please ignore it.
+func (na NetAddress) IsValid() bool {
+	_, _, err := net.SplitHostPort(string(na))
+	return err == nil
+}

modules/netaddress_test.go

@@ -0,0 +1,110 @@
+package modules
+
+import (
+	"testing"
+)
+
+// TestHost tests the Host method of the NetAddress type.
+func TestHost(t *testing.T) {
+	testSet := []struct {
+		query           NetAddress
+		desiredResponse string
+	}{
+		{"localhost", ""},
+		{"localhost:1234", "localhost"},
+		{"127.0.0.1", ""},
+		{"127.0.0.1:6723", "127.0.0.1"},
+		{"bbc.com", ""},
+		{"bbc.com:6434", "bbc.com"},
+		{"bitcoin.ninja", ""},
+		{"bitcoin.ninja:6752", "bitcoin.ninja"},
+		{"garbage:64:77", ""},
+		{"::1:5856", ""},
+		{"[::1]:5856", "::1"},
+		{"[::1]", ""},
+		{"::1", ""},
+	}
+	for _, test := range testSet {
+		if test.query.Host() != test.desiredResponse {
+			t.Error("test failed:", test, test.query.Host())
+		}
+	}
+}
+
+// TestIsLocal tests the IsLocal method of the NetAddress type.
+func TestIsLocal(t *testing.T) {
+	testSet := []struct {
+		query           NetAddress
+		desiredResponse bool
+	}{
+		// Networks such as 10.0.0.x have been omitted from testing - behavior
+		// for these networks is currently undefined.
+
+		// Localhost tests.
+		{"localhost", false},
+		{"localhost:1234", true},
+		{"127.0.0.1", false},
+		{"127.0.0.1:6723", true},
+		{"::1", false},
+		{"[::1]:7124", true},
+
+		// Public name tests.
+		{"hn.com", false},
+		{"hn.com:8811", false},
+		{"12.34.45.64", false},
+		{"12.34.45.64:7777", false},
+		{"::1:4646", false},
+
+		// Garbage name tests.
+		{"", false},
+		{"garbage", false},
+		{"garbage:6432", false},
+		{"garbage:6146:616", false},
+		{"[::1]", false},
+	}
+	for _, test := range testSet {
+		if test.query.IsLocal() != test.desiredResponse {
+			t.Error("test failed:", test, test.query.IsLocal())
+		}
+	}
+}
+
+// TestIsValid checks where a netaddress matches the regex for what counts as a
+// valid hostname or ip address.
+func TestIsValid(t *testing.T) {
+	testSet := []struct {
+		query           NetAddress
+		desiredResponse bool
+	}{
+		// Networks such as 10.0.0.x have been omitted from testing - behavior
+		// for these networks is currently undefined.
+
+		// Localhost tests.
+		{"localhost", false},
+		{"localhost:1234", true},
+		{"127.0.0.1", false},
+		{"127.0.0.1:6723", true},
+		{"::1", false},
+		{"[::1]:7124", true},
+
+		// Public name tests.
+		{"hn.com", false},
+		{"hn.com:8811", true},
+		{"12.34.45.64", false},
+		{"12.34.45.64:7777", true},
+		{"::1:4646", false},
+		{"plain", false},
+		{"plain:6432", true},
+
+		// Garbage name tests.
+		{"", false},
+		{"garbage:6146:616", false},
+		{"[::1]", false},
+		// {"google.com:notAPort", false}, TODO: Failed test case.
+	}
+	for _, test := range testSet {
+		if test.query.IsValid() != test.desiredResponse {
+			t.Error("test failed:", test, test.query.IsValid())
+		}
+	}
+}

modules/renter/files.go

@@ -135,7 +135,7 @@ func (r *Renter) DeleteFile(nickname string) error {
 	}
 	delete(r.files, nickname)
 
-	os.Remove(filepath.Join(r.saveDir, f.name+ShareExtension))
+	os.Remove(filepath.Join(r.persistDir, f.name+ShareExtension))
 
 	r.save()
 	return nil

modules/renter/persist.go

@@ -6,6 +6,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"io"
+	"log"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -124,7 +125,7 @@ func (f *file) load(r io.Reader) error {
 
 // saveFile saves a file to the renter directory.
 func (r *Renter) saveFile(f *file) error {
-	handle, err := persist.NewSafeFile(filepath.Join(r.saveDir, f.name+ShareExtension))
+	handle, err := persist.NewSafeFile(filepath.Join(r.persistDir, f.name+ShareExtension))
 	if err != nil {
 		return err
 	}
@@ -162,13 +163,13 @@ func (r *Renter) save() error {
 		b, _ := id.MarshalJSON()
 		data.Contracts[string(b)] = fc
 	}
-	return persist.SaveFile(saveMetadata, data, filepath.Join(r.saveDir, PersistFilename))
+	return persist.SaveFile(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename))
 }
 
 // load fetches the saved renter data from disk.
 func (r *Renter) load() error {
 	// Load all files found in renter directory.
-	dir, err := os.Open(r.saveDir) // TODO: store in a subdir?
+	dir, err := os.Open(r.persistDir) // TODO: store in a subdir?
 	if err != nil {
 		return err
 	}
@@ -181,7 +182,7 @@ func (r *Renter) load() error {
 		if filepath.Ext(path) != ShareExtension {
 			continue
 		}
-		file, err := os.Open(filepath.Join(r.saveDir, path))
+		file, err := os.Open(filepath.Join(r.persistDir, path))
 		if err != nil {
 			// maybe just skip?
 			return err
@@ -198,7 +199,7 @@ func (r *Renter) load() error {
 		Contracts map[string]types.FileContract
 		Entropy   [32]byte
 	}{}
-	err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.saveDir, PersistFilename))
+	err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.persistDir, PersistFilename))
 	if err != nil {
 		return err
 	}
@@ -346,6 +347,31 @@ func (r *Renter) loadSharedFiles(reader io.Reader) ([]string, error) {
 	return names, nil
 }
 
+// initPersist handles all of the persistence initialization, such as creating
+// the persistance directory and starting the logger.
+func (r *Renter) initPersist() error {
+	// Create the perist directory if it does not yet exist.
+	err := os.MkdirAll(r.persistDir, 0700)
+	if err != nil {
+		return err
+	}
+
+	// Initialize the logger.
+	logFile, err := os.OpenFile(filepath.Join(r.persistDir, "renter.log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
+	if err != nil {
+		return err
+	}
+	r.log = log.New(logFile, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile)
+	r.log.Println("STARTUP: Renter has started logging")
+
+	// Load the prior persistance structures.
+	err = r.load()
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
 // LoadSharedFiles loads a .sia file into the renter. It returns the nicknames
 // of the loaded files.
 func (r *Renter) LoadSharedFiles(filename string) ([]string, error) {

modules/renter/persist_test.go

@@ -158,7 +158,7 @@ func TestRenterSaveLoad(t *testing.T) {
 	}
 
 	// Corrupt a renter file and try to reload it.
-	err = ioutil.WriteFile(filepath.Join(rt.renter.saveDir, "corrupt"+ShareExtension), []byte{1, 2, 3}, 0660)
+	err = ioutil.WriteFile(filepath.Join(rt.renter.persistDir, "corrupt"+ShareExtension), []byte{1, 2, 3}, 0660)
 	if err != nil {
 		t.Fatal(err)
 	}

modules/renter/renter.go

@@ -3,7 +3,7 @@ package renter
 import (
 	"crypto/rand"
 	"errors"
-	"os"
+	"log"
 
 	"github.com/NebulousLabs/Sia/modules"
 	"github.com/NebulousLabs/Sia/sync"
@@ -30,13 +30,14 @@ type Renter struct {
 	contracts     map[types.FileContractID]types.FileContract
 	entropy       [32]byte // used to generate signing keys
 	downloadQueue []*download
-	saveDir       string
 
-	mu *sync.RWMutex
+	persistDir string
+	log        *log.Logger
+	mu         *sync.RWMutex
 }
 
 // New returns an empty renter.
-func New(cs modules.ConsensusSet, hdb modules.HostDB, wallet modules.Wallet, tpool modules.TransactionPool, saveDir string) (*Renter, error) {
+func New(cs modules.ConsensusSet, hdb modules.HostDB, wallet modules.Wallet, tpool modules.TransactionPool, persistDir string) (*Renter, error) {
 	if cs == nil {
 		return nil, ErrNilCS
 	}
@@ -58,25 +59,19 @@ func New(cs modules.ConsensusSet, hdb modules.HostDB, wallet modules.Wallet, tpo
 
 		files:     make(map[string]*file),
 		contracts: make(map[types.FileContractID]types.FileContract),
-		saveDir:   saveDir,
 
-		mu: sync.New(modules.SafeMutexDelay, 1),
+		persistDir: persistDir,
+		mu:         sync.New(modules.SafeMutexDelay, 1),
 	}
 	_, err := rand.Read(r.entropy[:])
 	if err != nil {
 		return nil, err
 	}
-
-	err = os.MkdirAll(saveDir, 0700)
+	err = r.initPersist()
 	if err != nil {
 		return nil, err
 	}
 
-	err = r.load()
-	if err != nil && !os.IsNotExist(err) {
-		return nil, err
-	}
-
 	r.cs.ConsensusSetSubscribe(r)
 
 	return r, nil