Fix --onion, RPC server and database

--onion option was broken earlier. Mirrored upstream change
btcsuite/btcd#446.

RPC server has now been updated to send objects to clients in
strictly sequential order of counters. This was accomplished
using heaps. RPC documentation has also been updated to reflect
the same.

Method signatures of a few database functions have been changed.
RemoveExpiredObjects now returns the list of objects that it
pruned from the database. This is going to be useful for the
object manager, to remove those objects from the list of
advertised objects. PubKey operations have been improved and
ciphering support (now a part of bmutil) has been integrated.
Tests for the same have also been added. Storage of public keys
has been optimized. Unencrypted and encrypted messages are now
stored separately. This is more efficient than storing everything
together.

README_RPC.md

@@ -32,9 +32,9 @@ func ReceivePubkey(object []byte, counter uint64)
 func ReceiveUnknownObject(object []byte, counter uint64)
 ```
 Receive objects of the given type from the server. Order of receiving objects
-(especially the initial unsynchronized ones) may be random. Client
-implementations must not rely on sequential values of counter. However, values
-of counter are guaranteed to be unique.
+is guaranteed to be sequential. Moreover, counter values for a particular object
+type are guaranteed to be unique. However, the same object may be sent twice
+(in a rare race condition).
 
 ## RPC Calls
 -----------

config.go

@@ -722,7 +722,7 @@ func loadConfig(isTest bool) (*config, []string, error) {
 // one was specified, but will otherwise use the normal dial function (which
 // could itself use a proxy or not).
 func bmdDial(network, address string) (net.Conn, error) {
-	if strings.HasSuffix(address, ".onion") {
+	if strings.Contains(address, ".onion:") {
 		return cfg.oniondial(network, address)
 	}
 	return cfg.dial(network, address)

database/db.go

@@ -103,12 +103,17 @@ type Db interface {
 	// RemoveExpiredObjects prunes all objects in the main circulation store
 	// whose expiry time has passed (along with a margin of 3 hours). This does
 	// not touch the pubkeys stored in the public key collection.
-	RemoveExpiredObjects() error
+	RemoveExpiredObjects() ([]*wire.ShaHash, error)
 
-	// RemovePubKey removes a PubKey from the PubKey store with the specified
-	// tag. Note that it doesn't touch the general object store and won't remove
-	// the public key from there.
-	RemovePubKey(*wire.ShaHash) error
+	// RemoveEncryptedPubKey removes a v4 PubKey with the specified tag from the
+	// encrypted PubKey store. Note that it doesn't touch the general object
+	// store and won't remove the public key from there.
+	RemoveEncryptedPubKey(*wire.ShaHash) error
+
+	// RemovePublicIdentity removes a v2/v3/unencrypted v4 public identity from
+	// the identity store. Note that it doesn't touch the general object store
+	// and won't remove the public key from there.
+	RemovePublicIdentity(*bmutil.Address) error
 
 	// RollbackClose discards the recent database changes to the previously
 	// saved data at last Sync and closes the database.

database/interface_test.go

@@ -9,6 +9,7 @@ package database_test
 
 import (
 	"bytes"
+	"encoding/hex"
 	"reflect"
 	"testing"
 	"time"
@@ -110,12 +111,12 @@ var testObj = [][]wire.Message{
 	[]wire.Message{
 		wire.NewMsgBroadcast(876, expires, 1, 1, &shahash[0],
 			[]byte{90, 87, 66, 45, 3, 2, 120, 101, 78, 78, 78, 7, 85, 55, 2, 23},
-			1, 1, 2, &pubkey[0], &pubkey[1], 3, 5, &ripehash[1], 1,
+			1, 1, 2, &pubkey[0], &pubkey[1], 3, 5, 1,
 			[]byte{27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41},
 			[]byte{42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56}),
 		wire.NewMsgBroadcast(876, expired, 1, 1, &shahash[1],
 			[]byte{90, 87, 66, 45, 3, 2, 120, 101, 78, 78, 78, 7, 85, 55},
-			1, 1, 2, &pubkey[2], &pubkey[3], 3, 5, &ripehash[0], 1,
+			1, 1, 2, &pubkey[2], &pubkey[3], 3, 5, 1,
 			[]byte{27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40},
 			[]byte{42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55}),
 	},
@@ -152,6 +153,13 @@ func testObject(tc *testContext) {
 				" got error %v", tc.dbType, i, err)
 		}
 
+		// Check for error on duplicate insertion.
+		_, err = tc.db.InsertObject(msg)
+		if err == nil {
+			tc.t.Errorf("InsertObject (%s): inserting duplicate object #%d,"+
+				" did not get error", tc.dbType, i)
+		}
+
 		hash := msg.InventoryHash()
 
 		exists, err := tc.db.ExistsObject(hash)
@@ -415,8 +423,8 @@ func testCounter(tc *testContext) {
 	}
 }
 
-// testPubKey tests inserting public key messages, FetchIdentityByAddress
-// and RemovePubKey
+// testPubKey tests inserting public key messages, FetchIdentityByAddress,
+// RemoveEncryptedPubKey and RemovePublicIdentity
 func testPubKey(tc *testContext) {
 	teardown := tc.newDb()
 	defer teardown()
@@ -451,34 +459,180 @@ func testPubKey(tc *testContext) {
 	}
 	_, err = tc.db.FetchIdentityByAddress(addr)
 	if err == nil {
-		tc.t.Fatalf("FetchIdentityByAddress (%s): expected error got none",
+		tc.t.Errorf("FetchIdentityByAddress (%s): expected error got none",
 			tc.dbType)
 	}
 
-	// test RemovePubKey for an address that does not exist
+	// test RemoveEncryptedPubKey for an address that does not exist
 	tag, _ := wire.NewShaHash(addr.Tag())
-	err = tc.db.RemovePubKey(tag)
+	err = tc.db.RemoveEncryptedPubKey(tag)
+	if err == nil {
+		tc.t.Errorf("RemoveEncryptedPubKey (%s): expected error got none",
+			tc.dbType)
+	}
+
+	// test RemovePublicIdentity for an address that does not exist
+	err = tc.db.RemovePublicIdentity(addr)
 	if err == nil {
-		tc.t.Fatalf("RemovePubKey (%s): expected error got none", tc.dbType)
+		tc.t.Errorf("RemovePublicIdentity (%s): expected error got none",
+			tc.dbType)
 	}
 
 	// test inserting valid v2 public key
+	addrV2, _ := bmutil.DecodeAddress("BM-orNprZ3PNHsLgK5CQMgRoje1aCKgRA4QP")
+	data, _ := hex.DecodeString("000000000000000000000000000000000000000102010000000187b541daefffc4b5ad1e61579e30c049709247630305e7962dcdf9ea2d0ef3e10ede4864ea5cfb42bd4ffa8b6f713490f106333b4e2ea8aed7b1ec7a7713958b380c6bac1f9742ef20fde832d0321a0643104ad98a91cf31f8ea0d28aa9886f19369ec065eafd823ae2c661cd586b111f72f18aa0b68db57f553f9b86f182f8f")
+	msg := new(wire.MsgObject)
+	err = msg.Decode(bytes.NewReader(data))
+	if err != nil {
+		tc.t.Fatal("failed to decode v2 pubkey, got error", err)
+	}
+
+	counter, err := tc.db.InsertObject(msg)
+	if err != nil {
+		tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
+	}
+	if counter != 2 {
+		tc.t.Errorf("InsertObject (%s): counter error, expected 2 got %d",
+			tc.dbType, counter)
+	}
+
 	// test inserting valid v3 public key
+	addrV3, _ := bmutil.DecodeAddress("BM-2D7oboD97WDibFcc792gkZjkvb3JQARiQx")
+	data, _ = hex.DecodeString("0000000001FB575F000000005581B73A00000001030100000001520A752F43BD36DA5BD2C77FDB7E53C597EB21BDA6BD08A80AC2F4ACC3D885DE19945F02D6D18A655FD831F071B6224E0F145F7C3138BE07DB7C4C9C8BD234DD8333DA6BA201B9893982B28B740AB6252E3A146677A1EDE15F567F15D8E8C83EAD7547AC132D008418330810243A43DBCF2DD39C5283913ED6BD6C1A3B468271FD03E8FD03E8473045022100AB37F26D1709E43FD24852273033D97764F2498E170422EDC6775FADE21F7A9502206FEB2527BBCAF77E7D07BAF6FCD2F4ED49B8B4D1C3FCE7DEB6149D7E9DF3CD95")
+	msg = new(wire.MsgObject)
+	err = msg.Decode(bytes.NewReader(data))
+	if err != nil {
+		tc.t.Fatal("failed to decode v3 pubkey, got error", err)
+	}
+
+	counter, err = tc.db.InsertObject(msg)
+	if err != nil {
+		tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
+	}
+	if counter != 3 {
+		tc.t.Errorf("InsertObject (%s): counter error, expected 3 got %d",
+			tc.dbType, counter)
+	}
+
 	// test inserting valid v4 public key
+	addrV4, _ := bmutil.DecodeAddress("BM-2cTFEueNqmjgR3EqduEZmaZbEW1h9z7M7o")
+	data, _ = hex.DecodeString("00000000025A04D60000000055A4EA7C0000000104017F933D64A866DE24C27D647C74068A59DCEE0CABFC1DF887BE7DD30BA3BD9143D513F0B37087891F6A98DD0B55B1A73E02CA002090CE6A050E760F52D18F7F50B1B9139DBCEF861254C195173AA601DE8A72B52E00206FAF91EDD32E213097CD91E4ACBB883CB2F8CC6AAFCC670DDE1FAC52C210469D71B08A162E07C4B8926A50CC0701594AF55D65052C2D9D74CE28BB571D781423C101BDC8DB6CE3FA639BDDE9CE39364307188470AEC410F7EE2BCC008CA6B1F2A37CF0841FC5EDE154C172438061577FBF3BC6BCDAAAB9BBCC90378DE815A99B0B78D81DFC9ABE33F99B4BC2AFAC2101ED7E0E213C00011FF3583B1E2BAADEF4BED2DB17F340258C22D38F8B490040B94E01F76F2118D90D718FFAFFB7D8F2A9F2B3498D45D528F16BCE55B43E63AAF3AED720F0AC06FCEB853661ACE13714069AA47A3D2FD6180AD0458B344E7AF04A26A25490DCEF236EE29CDF2FD96CDF55EB2B0D4DACA1EC21B4049DB6A6C713A2350D6ECE4C77C01DA01BCAAB2F2CBB31")
+	msg = new(wire.MsgObject)
+	err = msg.Decode(bytes.NewReader(data))
+	if err != nil {
+		tc.t.Fatal("failed to decode v4 pubkey, got error", err)
+	}
+
+	counter, err = tc.db.InsertObject(msg)
+	if err != nil {
+		tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
+	}
+	if counter != 4 {
+		tc.t.Errorf("InsertObject (%s): count error, expected 4 got %d",
+			tc.dbType, counter)
+	}
 
 	// test FetchIdentityByAddress for an address that exists in the database
 	// v2 address
+	idV2, err := tc.db.FetchIdentityByAddress(addrV2)
+	if err != nil {
+		tc.t.Errorf("FetchIdentityByAddress (%s): got error %v", tc.dbType,
+			err)
+	}
+	if !reflect.DeepEqual(&idV2.Address, addrV2) {
+		tc.t.Errorf("FetchIdentityByAddress (%s): identities not equal",
+			tc.dbType)
+	}
+
 	// v3 address
+	idV3, err := tc.db.FetchIdentityByAddress(addrV3)
+	if err != nil {
+		tc.t.Errorf("FetchIdentityByAddress (%s): got error %v", tc.dbType,
+			err)
+	}
+	if !reflect.DeepEqual(&idV3.Address, addrV3) {
+		tc.t.Errorf("FetchIdentityByAddress (%s): identities not equal",
+			tc.dbType)
+	}
+
 	// v4 address
+	idV4, err := tc.db.FetchIdentityByAddress(addrV4)
+	if err != nil {
+		tc.t.Errorf("FetchIdentityByAddress (%s): got error %v", tc.dbType,
+			err)
+	}
+	if !reflect.DeepEqual(&idV4.Address, addrV4) {
+		tc.t.Errorf("FetchIdentityByAddress (%s): identities not equal",
+			tc.dbType)
+	}
 
-	// test RemovePubKey for an address that exists in the database
+	// Test whether RemoveEncryptedPubKey fails for a decrypted key.
+	tag, _ = wire.NewShaHash(idV4.Tag())
+	err = tc.db.RemoveEncryptedPubKey(tag)
+	if err != database.ErrNonexistentObject {
+		tc.t.Errorf("RemoveEncryptedPubKey (%s): expected nonexistent object"+
+			" error, got %v", tc.dbType, err)
+	}
+
+	// Test RemoveEncryptedPubKey for a tag that exists in the database
+	data, _ = hex.DecodeString("0000000000A229F300000000559BAFB0000000010401F17457B60376F21C20F485652A8A3047ECBBAD4E609E3594D74C7A94D55B994BB4385338FF5E76639EA4EB083C01CCE402CA00205BA2AE2C4519376892C8352EE978E91B298071FFC2F9A211B566715C1D8A4EFF002018E8708FACA508F037DB07E05015B3F91BAAAE22F6A5FF24C4F0C9AF3B1119B01AE14110E14ECC4B511D867DDA589F26A257AD07244F1BCA502C2BFF4269AF29F99C2437C555F4CC42FF510607E4787082354D6C4CDD437010EFDC9B7C88783855988578890C5CBB1FBAE692B7B40C6E1B4642299E181BADCB48D9F931701BA5351161C7E136703DDE5D85735F6366184195C68EF0FB401304B068ED10E785B78C503A3D15FE5D302B0D660E611004BC2925D45D4F65F6E9607F5EA5CA64CBF87B753318AC55364D471B589FFD946E85AC8BA3B6FE82E71A34403852C55A4F4CDCECB89F77032AB8BF18D7C68A983CB902456C90367D3494FA040A69830F4A140E251BCD78A7A36E5F674D5B91C553EF2A3DA17D0F1E9A05F154719F39CBEEF1")
+	msg = new(wire.MsgObject)
+	err = msg.Decode(bytes.NewReader(data))
+	if err != nil {
+		tc.t.Fatal("failed to decode v4 pubkey, got error", err)
+	}
+	_, err = tc.db.InsertObject(msg)
+	if err != nil {
+		tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
+	}
+	tag, _ = wire.NewShaHash(msg.Payload[:32])
+	err = tc.db.RemoveEncryptedPubKey(tag)
+	if err != nil {
+		tc.t.Errorf("RemoveEncryptedPubKey (%s): got error %v", tc.dbType, err)
+	}
+
+	// test RemovePublicIdentity for an address that exists in the database
+	// v2
+	err = tc.db.RemovePublicIdentity(addrV2)
+	if err != nil {
+		tc.t.Errorf("RemovePublicIdentity (%s): got error %v", err)
+	}
+	// v3
+	err = tc.db.RemovePublicIdentity(addrV3)
+	if err != nil {
+		tc.t.Errorf("RemovePublicIdentity (%s): got error %v", err)
+	}
+	// decrypted v4
+	err = tc.db.RemovePublicIdentity(addrV4)
+	if err != nil {
+		tc.t.Errorf("RemovePublicIdentity (%s): got error %v", err)
+	}
+	// Test RemovePubKey for an address that was removed.
+	err = tc.db.RemovePublicIdentity(addrV2)
+	if err != database.ErrNonexistentObject {
+		tc.t.Errorf("RemovePublicIdentity (%s): expected nonexistent object"+
+			" error, got %v", tc.dbType, err)
+	}
 
 	// test FetchIdentityByAddress for an address that was removed
 	// v2 address
+	_, err = tc.db.FetchIdentityByAddress(addrV2)
+	if err != database.ErrNonexistentObject {
+		tc.t.Errorf("FetchIdentityByAddress (%s): expected nonexistent object"+
+			" error, got %v", tc.dbType, err)
+	}
 	// v3 address
+	_, err = tc.db.FetchIdentityByAddress(addrV3)
+	if err != database.ErrNonexistentObject {
+		tc.t.Errorf("FetchIdentityByAddress (%s): expected nonexistent object"+
+			" error, got %v", tc.dbType, err)
+	}
 	// v4 address
-
-	// test RemovePubKey for an address that was removed
+	_, err = tc.db.FetchIdentityByAddress(addrV4)
+	if err != database.ErrNonexistentObject {
+		tc.t.Errorf("FetchIdentityByAddress (%s): expected nonexistent object"+
+			" error, got %v", tc.dbType, err)
+	}
 }
 
 // tests FetchRandomInvHashes and FilterObjects