diff --git a/Makefile b/Makefile index c85116ef10..1bb63cf4ba 100644 --- a/Makefile +++ b/Makefile @@ -62,11 +62,10 @@ clean: # command. 'test' runs short tests that should last no more than a few seconds, # 'test-long' runs more thorough tests which should not last more than a few # minutes. -pkgs = ./api ./compatibility ./crypto ./encoding ./modules/consensus \ - ./modules/gateway ./modules/host ./modules/hostdb \ - ./modules/miner ./modules/renter ./modules/transactionpool \ - ./modules/wallet ./modules/explorer ./persist \ - ./siae ./types +pkgs = ./api ./compatibility ./crypto ./encoding ./modules ./modules/consensus \ + ./modules/explorer ./modules/gateway ./modules/host ./modules/hostdb \ + ./modules/miner ./modules/renter ./modules/transactionpool \ + ./modules/wallet ./persist ./siae ./types test: clean fmt REBUILD go test -short -tags='debug testing' -timeout=10s $(pkgs) test-v: clean fmt REBUILD diff --git a/modules/gateway.go b/modules/gateway.go index eec069aec8..07f4c756de 100644 --- a/modules/gateway.go +++ b/modules/gateway.go @@ -32,21 +32,6 @@ type PeerConn interface { // keeping the connection open after all necessary I/O has been performed. type RPCFunc func(PeerConn) error -// A NetAddress contains the information needed to contact a peer. -type NetAddress string - -// Host returns the NetAddress' IP. -func (na NetAddress) Host() string { - host, _, _ := net.SplitHostPort(string(na)) - return host -} - -// Port returns the NetAddress' port number. -func (na NetAddress) Port() string { - _, port, _ := net.SplitHostPort(string(na)) - return port -} - // A Gateway facilitates the interactions between the local node and remote // nodes (peers). It relays incoming blocks and transactions to local modules, // and broadcasts outgoing blocks and transactions to peers. In a broad sense, diff --git a/modules/hostdb/hostentry.go b/modules/hostdb/hostentry.go index e902eebfd8..63999107c5 100644 --- a/modules/hostdb/hostentry.go +++ b/modules/hostdb/hostentry.go @@ -1,8 +1,7 @@ package hostdb import ( - "net" - + "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/types" ) @@ -17,11 +16,14 @@ type hostEntry struct { // insert adds a host entry to the state. The host will be inserted into the // set of all hosts, and if it is online and responding to requests it will be // put into the list of active hosts. +// +// TODO: Function should return an error. func (hdb *HostDB) insertHost(host modules.HostSettings) { - // Blacklist localhost. - if host.IPAddress.Host() == "localhost" { + // Remove garbage hosts and local hosts. + if !host.IPAddress.IsValid() { return - } else if ip := net.ParseIP(string(host.IPAddress)); ip != nil && ip.IsLoopback() { + } + if host.IPAddress.IsLocal() && build.Release != "testing" { return } diff --git a/modules/netaddress.go b/modules/netaddress.go new file mode 100644 index 0000000000..2c7cb34c4a --- /dev/null +++ b/modules/netaddress.go @@ -0,0 +1,52 @@ +package modules + +import ( + "net" +) + +// A NetAddress contains the information needed to contact a peer. +type NetAddress string + +// Host removes the port from a NetAddress, returning just the host. If the +// address is invalid, the empty string is returned. +func (na NetAddress) Host() string { + host, _, err := net.SplitHostPort(string(na)) + // 'host' is not always the empty string if an error is returned. + if err != nil { + return "" + } + return host +} + +// Port returns the NetAddress object's port number. The empty string is +// returned if the NetAddress is invalid. +func (na NetAddress) Port() string { + _, port, err := net.SplitHostPort(string(na)) + // 'port' will not always be the empty string if an error is returned. + if err != nil { + return "" + } + return port +} + +// IsLocal returns true for ip addresses that are on the same machine. +func (na NetAddress) IsLocal() bool { + if !na.IsValid() { + return false + } + + host := na.Host() + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + return true + } + if host == "localhost" { + return true + } + return false +} + +// IsValid does nothing. Please ignore it. +func (na NetAddress) IsValid() bool { + _, _, err := net.SplitHostPort(string(na)) + return err == nil +} diff --git a/modules/netaddress_test.go b/modules/netaddress_test.go new file mode 100644 index 0000000000..e4abb02409 --- /dev/null +++ b/modules/netaddress_test.go @@ -0,0 +1,110 @@ +package modules + +import ( + "testing" +) + +// TestHost tests the Host method of the NetAddress type. +func TestHost(t *testing.T) { + testSet := []struct { + query NetAddress + desiredResponse string + }{ + {"localhost", ""}, + {"localhost:1234", "localhost"}, + {"127.0.0.1", ""}, + {"127.0.0.1:6723", "127.0.0.1"}, + {"bbc.com", ""}, + {"bbc.com:6434", "bbc.com"}, + {"bitcoin.ninja", ""}, + {"bitcoin.ninja:6752", "bitcoin.ninja"}, + {"garbage:64:77", ""}, + {"::1:5856", ""}, + {"[::1]:5856", "::1"}, + {"[::1]", ""}, + {"::1", ""}, + } + for _, test := range testSet { + if test.query.Host() != test.desiredResponse { + t.Error("test failed:", test, test.query.Host()) + } + } +} + +// TestIsLocal tests the IsLocal method of the NetAddress type. +func TestIsLocal(t *testing.T) { + testSet := []struct { + query NetAddress + desiredResponse bool + }{ + // Networks such as 10.0.0.x have been omitted from testing - behavior + // for these networks is currently undefined. + + // Localhost tests. + {"localhost", false}, + {"localhost:1234", true}, + {"127.0.0.1", false}, + {"127.0.0.1:6723", true}, + {"::1", false}, + {"[::1]:7124", true}, + + // Public name tests. + {"hn.com", false}, + {"hn.com:8811", false}, + {"12.34.45.64", false}, + {"12.34.45.64:7777", false}, + {"::1:4646", false}, + + // Garbage name tests. + {"", false}, + {"garbage", false}, + {"garbage:6432", false}, + {"garbage:6146:616", false}, + {"[::1]", false}, + } + for _, test := range testSet { + if test.query.IsLocal() != test.desiredResponse { + t.Error("test failed:", test, test.query.IsLocal()) + } + } +} + +// TestIsValid checks where a netaddress matches the regex for what counts as a +// valid hostname or ip address. +func TestIsValid(t *testing.T) { + testSet := []struct { + query NetAddress + desiredResponse bool + }{ + // Networks such as 10.0.0.x have been omitted from testing - behavior + // for these networks is currently undefined. + + // Localhost tests. + {"localhost", false}, + {"localhost:1234", true}, + {"127.0.0.1", false}, + {"127.0.0.1:6723", true}, + {"::1", false}, + {"[::1]:7124", true}, + + // Public name tests. + {"hn.com", false}, + {"hn.com:8811", true}, + {"12.34.45.64", false}, + {"12.34.45.64:7777", true}, + {"::1:4646", false}, + {"plain", false}, + {"plain:6432", true}, + + // Garbage name tests. + {"", false}, + {"garbage:6146:616", false}, + {"[::1]", false}, + // {"google.com:notAPort", false}, TODO: Failed test case. + } + for _, test := range testSet { + if test.query.IsValid() != test.desiredResponse { + t.Error("test failed:", test, test.query.IsValid()) + } + } +} diff --git a/modules/renter/files.go b/modules/renter/files.go index 499835f37c..17ef404093 100644 --- a/modules/renter/files.go +++ b/modules/renter/files.go @@ -135,7 +135,7 @@ func (r *Renter) DeleteFile(nickname string) error { } delete(r.files, nickname) - os.Remove(filepath.Join(r.saveDir, f.name+ShareExtension)) + os.Remove(filepath.Join(r.persistDir, f.name+ShareExtension)) r.save() return nil diff --git a/modules/renter/persist.go b/modules/renter/persist.go index bfc8d59a5e..f3bb01f529 100644 --- a/modules/renter/persist.go +++ b/modules/renter/persist.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "io" + "log" "os" "path/filepath" "strconv" @@ -124,7 +125,7 @@ func (f *file) load(r io.Reader) error { // saveFile saves a file to the renter directory. func (r *Renter) saveFile(f *file) error { - handle, err := persist.NewSafeFile(filepath.Join(r.saveDir, f.name+ShareExtension)) + handle, err := persist.NewSafeFile(filepath.Join(r.persistDir, f.name+ShareExtension)) if err != nil { return err } @@ -162,13 +163,13 @@ func (r *Renter) save() error { b, _ := id.MarshalJSON() data.Contracts[string(b)] = fc } - return persist.SaveFile(saveMetadata, data, filepath.Join(r.saveDir, PersistFilename)) + return persist.SaveFile(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename)) } // load fetches the saved renter data from disk. func (r *Renter) load() error { // Load all files found in renter directory. - dir, err := os.Open(r.saveDir) // TODO: store in a subdir? + dir, err := os.Open(r.persistDir) // TODO: store in a subdir? if err != nil { return err } @@ -181,7 +182,7 @@ func (r *Renter) load() error { if filepath.Ext(path) != ShareExtension { continue } - file, err := os.Open(filepath.Join(r.saveDir, path)) + file, err := os.Open(filepath.Join(r.persistDir, path)) if err != nil { // maybe just skip? return err @@ -198,7 +199,7 @@ func (r *Renter) load() error { Contracts map[string]types.FileContract Entropy [32]byte }{} - err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.saveDir, PersistFilename)) + err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.persistDir, PersistFilename)) if err != nil { return err } @@ -346,6 +347,31 @@ func (r *Renter) loadSharedFiles(reader io.Reader) ([]string, error) { return names, nil } +// initPersist handles all of the persistence initialization, such as creating +// the persistance directory and starting the logger. +func (r *Renter) initPersist() error { + // Create the perist directory if it does not yet exist. + err := os.MkdirAll(r.persistDir, 0700) + if err != nil { + return err + } + + // Initialize the logger. + logFile, err := os.OpenFile(filepath.Join(r.persistDir, "renter.log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660) + if err != nil { + return err + } + r.log = log.New(logFile, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile) + r.log.Println("STARTUP: Renter has started logging") + + // Load the prior persistance structures. + err = r.load() + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + // LoadSharedFiles loads a .sia file into the renter. It returns the nicknames // of the loaded files. func (r *Renter) LoadSharedFiles(filename string) ([]string, error) { diff --git a/modules/renter/persist_test.go b/modules/renter/persist_test.go index e1329b9950..893c893001 100644 --- a/modules/renter/persist_test.go +++ b/modules/renter/persist_test.go @@ -158,7 +158,7 @@ func TestRenterSaveLoad(t *testing.T) { } // Corrupt a renter file and try to reload it. - err = ioutil.WriteFile(filepath.Join(rt.renter.saveDir, "corrupt"+ShareExtension), []byte{1, 2, 3}, 0660) + err = ioutil.WriteFile(filepath.Join(rt.renter.persistDir, "corrupt"+ShareExtension), []byte{1, 2, 3}, 0660) if err != nil { t.Fatal(err) } diff --git a/modules/renter/renter.go b/modules/renter/renter.go index 16224dc743..5a17bd45cd 100644 --- a/modules/renter/renter.go +++ b/modules/renter/renter.go @@ -3,7 +3,7 @@ package renter import ( "crypto/rand" "errors" - "os" + "log" "github.com/NebulousLabs/Sia/modules" "github.com/NebulousLabs/Sia/sync" @@ -30,13 +30,14 @@ type Renter struct { contracts map[types.FileContractID]types.FileContract entropy [32]byte // used to generate signing keys downloadQueue []*download - saveDir string - mu *sync.RWMutex + persistDir string + log *log.Logger + mu *sync.RWMutex } // New returns an empty renter. -func New(cs modules.ConsensusSet, hdb modules.HostDB, wallet modules.Wallet, tpool modules.TransactionPool, saveDir string) (*Renter, error) { +func New(cs modules.ConsensusSet, hdb modules.HostDB, wallet modules.Wallet, tpool modules.TransactionPool, persistDir string) (*Renter, error) { if cs == nil { return nil, ErrNilCS } @@ -58,25 +59,19 @@ func New(cs modules.ConsensusSet, hdb modules.HostDB, wallet modules.Wallet, tpo files: make(map[string]*file), contracts: make(map[types.FileContractID]types.FileContract), - saveDir: saveDir, - mu: sync.New(modules.SafeMutexDelay, 1), + persistDir: persistDir, + mu: sync.New(modules.SafeMutexDelay, 1), } _, err := rand.Read(r.entropy[:]) if err != nil { return nil, err } - - err = os.MkdirAll(saveDir, 0700) + err = r.initPersist() if err != nil { return nil, err } - err = r.load() - if err != nil && !os.IsNotExist(err) { - return nil, err - } - r.cs.ConsensusSetSubscribe(r) return r, nil diff --git a/modules/renter/renter_test.go b/modules/renter/renter_test.go index 2a224a341f..8d7b042f93 100644 --- a/modules/renter/renter_test.go +++ b/modules/renter/renter_test.go @@ -128,27 +128,27 @@ func TestNilInputs(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = New(rt.cs, rt.hostdb, rt.wallet, rt.tpool, rt.renter.saveDir+"1") + _, err = New(rt.cs, rt.hostdb, rt.wallet, rt.tpool, rt.renter.persistDir+"1") if err != nil { t.Error(err) } - _, err = New(nil, nil, nil, nil, rt.renter.saveDir+"2") + _, err = New(nil, nil, nil, nil, rt.renter.persistDir+"2") if err == nil { t.Error("no error returned for nil inputs") } - _, err = New(nil, rt.hostdb, rt.wallet, rt.tpool, rt.renter.saveDir+"3") + _, err = New(nil, rt.hostdb, rt.wallet, rt.tpool, rt.renter.persistDir+"3") if err != ErrNilCS { t.Error(err) } - _, err = New(rt.cs, nil, rt.wallet, rt.tpool, rt.renter.saveDir+"5") + _, err = New(rt.cs, nil, rt.wallet, rt.tpool, rt.renter.persistDir+"5") if err != ErrNilHostDB { t.Error(err) } - _, err = New(rt.cs, rt.hostdb, nil, rt.tpool, rt.renter.saveDir+"6") + _, err = New(rt.cs, rt.hostdb, nil, rt.tpool, rt.renter.persistDir+"6") if err != ErrNilWallet { t.Error(err) } - _, err = New(rt.cs, rt.hostdb, rt.wallet, nil, rt.renter.saveDir+"6") + _, err = New(rt.cs, rt.hostdb, rt.wallet, nil, rt.renter.persistDir+"6") if err != ErrNilTpool { t.Error(err) } diff --git a/modules/renter/upload.go b/modules/renter/upload.go index 37b193eb9b..45d1b1f364 100644 --- a/modules/renter/upload.go +++ b/modules/renter/upload.go @@ -183,12 +183,12 @@ func (r *Renter) Upload(up modules.FileUploadParams) error { totalsize := up.PieceSize * uint64(up.ErasureCode.NumPieces()) * f.numChunks() var hosts []uploader for _, host := range r.hostDB.RandomHosts(up.ErasureCode.NumPieces() * 3 / 2) { - host, err := r.newHostUploader(host, totalsize, up.Duration, f.masterKey) + hostUploader, err := r.newHostUploader(host, totalsize, up.Duration, f.masterKey) if err != nil { continue } - defer host.Close() - hosts = append(hosts, host) + defer hostUploader.Close() + hosts = append(hosts, hostUploader) } if len(hosts) < up.ErasureCode.MinPieces() { return errors.New("not enough hosts to support upload")