Skip to content

Commit

Permalink
Fix --onion, RPC server and database
Browse files Browse the repository at this point in the history 
--onion option was broken earlier. Mirrored upstream change
btcsuite/btcd#446.

RPC server has now been updated to send objects to clients in strictly
sequential order of counters. This was accomplished using heaps. RPC
documentation has also been updated to reflect the same.

Method signatures of a few database functions have been changed.
RemoveExpiredObjects now returns the list of objects that it pruned
from the database. This is going to be useful for the object manager,
to remove those objects from the list of advertised objects. PubKey
operations have been improved and ciphering support (now a part of
bmutil) has been integrated. Tests for the same have also been added.
Storage of public keys has been optimized. Unencrypted and encrypted
messages are now stored separately. This is more efficient than storing
everything together.
  • Loading branch information
@ishbir
ishbir committed Jun 23, 2015
1 parent e33ed8e commit ffbccea
Show file tree
Hide file tree
Showing 11 changed files with 533 additions and 165 deletions.
6 changes: 3 additions & 3 deletions README_RPC.md
View file
Expand Up @@ -32,9 +32,9 @@ func ReceivePubkey(object []byte, counter uint64)
func ReceiveUnknownObject(object []byte, counter uint64)
```
Receive objects of the given type from the server. Order of receiving objects
(especially the initial unsynchronized ones) may be random. Client
implementations must not rely on sequential values of counter. However, values
of counter are guaranteed to be unique.
is guaranteed to be sequential. Moreover, counter values for a particular object
type are guaranteed to be unique. However, the same object may be sent twice
(in a rare race condition).

## RPC Calls
-----------
Expand Down
2 changes: 1 addition & 1 deletion config.go
View file
Expand Up @@ -722,7 +722,7 @@ func loadConfig(isTest bool) (*config, []string, error) {
// one was specified, but will otherwise use the normal dial function (which
// could itself use a proxy or not).
func bmdDial(network, address string) (net.Conn, error) {
if strings.HasSuffix(address, ".onion") {
if strings.Contains(address, ".onion:") {
return cfg.oniondial(network, address)
}
return cfg.dial(network, address)
Expand Down
18 changes: 12 additions & 6 deletions database/db.go
View file
Expand Up @@ -103,12 +103,18 @@ type Db interface {
// RemoveExpiredObjects prunes all objects in the main circulation store
// whose expiry time has passed (along with a margin of 3 hours). This does
// not touch the pubkeys stored in the public key collection.
RemoveExpiredObjects() error

// RemovePubKey removes a PubKey from the PubKey store with the specified
// tag. Note that it doesn't touch the general object store and won't remove
// the public key from there.
RemovePubKey(*wire.ShaHash) error
RemoveExpiredObjects() ([]*wire.ShaHash, error)

// RemoveEncryptedPubKey removes a v4 PubKey with the specified tag from the
// encrypted PubKey store. Note that it doesn't touch the general object
// store and won't remove the public key from there.
RemoveEncryptedPubKey(*wire.ShaHash) error

// RemovePublicIdentity removes the public identity corresponding the given
// address from the database. This includes any v2/v3/previously used v4
// identities. Note that it doesn't touch the general object store and won't
// remove the public key object from there.
RemovePublicIdentity(*bmutil.Address) error

// RollbackClose discards the recent database changes to the previously
// saved data at last Sync and closes the database.
Expand Down
176 changes: 165 additions & 11 deletions database/interface_test.go
View file
Expand Up @@ -9,6 +9,7 @@ package database_test

import (
"bytes"
"encoding/hex"
"reflect"
"testing"
"time"
Expand Down Expand Up @@ -110,12 +111,12 @@ var testObj = [][]wire.Message{
[]wire.Message{
wire.NewMsgBroadcast(876, expires, 1, 1, &shahash[0],
[]byte{90, 87, 66, 45, 3, 2, 120, 101, 78, 78, 78, 7, 85, 55, 2, 23},
1, 1, 2, &pubkey[0], &pubkey[1], 3, 5, &ripehash[1], 1,
1, 1, 2, &pubkey[0], &pubkey[1], 3, 5, 1,
[]byte{27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41},
[]byte{42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56}),
wire.NewMsgBroadcast(876, expired, 1, 1, &shahash[1],
[]byte{90, 87, 66, 45, 3, 2, 120, 101, 78, 78, 78, 7, 85, 55},
1, 1, 2, &pubkey[2], &pubkey[3], 3, 5, &ripehash[0], 1,
1, 1, 2, &pubkey[2], &pubkey[3], 3, 5, 1,
[]byte{27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40},
[]byte{42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55}),
},
Expand Down Expand Up @@ -152,6 +153,13 @@ func testObject(tc *testContext) {
" got error %v", tc.dbType, i, err)
}

// Check for error on duplicate insertion.
_, err = tc.db.InsertObject(msg)
if err == nil {
tc.t.Errorf("InsertObject (%s): inserting duplicate object #%d,"+
" did not get error", tc.dbType, i)
}

hash := msg.InventoryHash()

exists, err := tc.db.ExistsObject(hash)
Expand Down Expand Up @@ -415,8 +423,8 @@ func testCounter(tc *testContext) {
}
}

// testPubKey tests inserting public key messages, FetchIdentityByAddress
// and RemovePubKey
// testPubKey tests inserting public key messages, FetchIdentityByAddress,
// RemoveEncryptedPubKey and RemovePublicIdentity
func testPubKey(tc *testContext) {
teardown := tc.newDb()
defer teardown()
Expand Down Expand Up @@ -451,34 +459,180 @@ func testPubKey(tc *testContext) {
}
_, err = tc.db.FetchIdentityByAddress(addr)
if err == nil {
tc.t.Fatalf("FetchIdentityByAddress (%s): expected error got none",
tc.t.Errorf("FetchIdentityByAddress (%s): expected error got none",
tc.dbType)
}

// test RemovePubKey for an address that does not exist
// test RemoveEncryptedPubKey for an address that does not exist
tag, _ := wire.NewShaHash(addr.Tag())
err = tc.db.RemovePubKey(tag)
err = tc.db.RemoveEncryptedPubKey(tag)
if err == nil {
tc.t.Errorf("RemoveEncryptedPubKey (%s): expected error got none",
tc.dbType)
}

// test RemovePublicIdentity for an address that does not exist
err = tc.db.RemovePublicIdentity(addr)
if err == nil {
tc.t.Fatalf("RemovePubKey (%s): expected error got none", tc.dbType)
tc.t.Errorf("RemovePublicIdentity (%s): expected error got none",
tc.dbType)
}

// test inserting valid v2 public key
addrV2, _ := bmutil.DecodeAddress("BM-orNprZ3PNHsLgK5CQMgRoje1aCKgRA4QP")
data, _ := hex.DecodeString("000000000000000000000000000000000000000102010000000187b541daefffc4b5ad1e61579e30c049709247630305e7962dcdf9ea2d0ef3e10ede4864ea5cfb42bd4ffa8b6f713490f106333b4e2ea8aed7b1ec7a7713958b380c6bac1f9742ef20fde832d0321a0643104ad98a91cf31f8ea0d28aa9886f19369ec065eafd823ae2c661cd586b111f72f18aa0b68db57f553f9b86f182f8f")
msg := new(wire.MsgObject)
err = msg.Decode(bytes.NewReader(data))
if err != nil {
tc.t.Fatal("failed to decode v2 pubkey, got error", err)
}

counter, err := tc.db.InsertObject(msg)
if err != nil {
tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
}
if counter != 2 {
tc.t.Errorf("InsertObject (%s): counter error, expected 2 got %d",
tc.dbType, counter)
}

// test inserting valid v3 public key
addrV3, _ := bmutil.DecodeAddress("BM-2D7oboD97WDibFcc792gkZjkvb3JQARiQx")
data, _ = hex.DecodeString("0000000001FB575F000000005581B73A00000001030100000001520A752F43BD36DA5BD2C77FDB7E53C597EB21BDA6BD08A80AC2F4ACC3D885DE19945F02D6D18A655FD831F071B6224E0F145F7C3138BE07DB7C4C9C8BD234DD8333DA6BA201B9893982B28B740AB6252E3A146677A1EDE15F567F15D8E8C83EAD7547AC132D008418330810243A43DBCF2DD39C5283913ED6BD6C1A3B468271FD03E8FD03E8473045022100AB37F26D1709E43FD24852273033D97764F2498E170422EDC6775FADE21F7A9502206FEB2527BBCAF77E7D07BAF6FCD2F4ED49B8B4D1C3FCE7DEB6149D7E9DF3CD95")
msg = new(wire.MsgObject)
err = msg.Decode(bytes.NewReader(data))
if err != nil {
tc.t.Fatal("failed to decode v3 pubkey, got error", err)
}

counter, err = tc.db.InsertObject(msg)
if err != nil {
tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
}
if counter != 3 {
tc.t.Errorf("InsertObject (%s): counter error, expected 3 got %d",
tc.dbType, counter)
}

// test inserting valid v4 public key
addrV4, _ := bmutil.DecodeAddress("BM-2cTFEueNqmjgR3EqduEZmaZbEW1h9z7M7o")
data, _ = hex.DecodeString("00000000025A04D60000000055A4EA7C0000000104017F933D64A866DE24C27D647C74068A59DCEE0CABFC1DF887BE7DD30BA3BD9143D513F0B37087891F6A98DD0B55B1A73E02CA002090CE6A050E760F52D18F7F50B1B9139DBCEF861254C195173AA601DE8A72B52E00206FAF91EDD32E213097CD91E4ACBB883CB2F8CC6AAFCC670DDE1FAC52C210469D71B08A162E07C4B8926A50CC0701594AF55D65052C2D9D74CE28BB571D781423C101BDC8DB6CE3FA639BDDE9CE39364307188470AEC410F7EE2BCC008CA6B1F2A37CF0841FC5EDE154C172438061577FBF3BC6BCDAAAB9BBCC90378DE815A99B0B78D81DFC9ABE33F99B4BC2AFAC2101ED7E0E213C00011FF3583B1E2BAADEF4BED2DB17F340258C22D38F8B490040B94E01F76F2118D90D718FFAFFB7D8F2A9F2B3498D45D528F16BCE55B43E63AAF3AED720F0AC06FCEB853661ACE13714069AA47A3D2FD6180AD0458B344E7AF04A26A25490DCEF236EE29CDF2FD96CDF55EB2B0D4DACA1EC21B4049DB6A6C713A2350D6ECE4C77C01DA01BCAAB2F2CBB31")
msg = new(wire.MsgObject)
err = msg.Decode(bytes.NewReader(data))
if err != nil {
tc.t.Fatal("failed to decode v4 pubkey, got error", err)
}

counter, err = tc.db.InsertObject(msg)
if err != nil {
tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
}
if counter != 4 {
tc.t.Errorf("InsertObject (%s): count error, expected 4 got %d",
tc.dbType, counter)
}

// test FetchIdentityByAddress for an address that exists in the database
// v2 address
idV2, err := tc.db.FetchIdentityByAddress(addrV2)
if err != nil {
tc.t.Errorf("FetchIdentityByAddress (%s): got error %v", tc.dbType,
err)
}
if !reflect.DeepEqual(&idV2.Address, addrV2) {
tc.t.Errorf("FetchIdentityByAddress (%s): identities not equal",
tc.dbType)
}

// v3 address
idV3, err := tc.db.FetchIdentityByAddress(addrV3)
if err != nil {
tc.t.Errorf("FetchIdentityByAddress (%s): got error %v", tc.dbType,
err)
}
if !reflect.DeepEqual(&idV3.Address, addrV3) {
tc.t.Errorf("FetchIdentityByAddress (%s): identities not equal",
tc.dbType)
}

// v4 address
idV4, err := tc.db.FetchIdentityByAddress(addrV4)
if err != nil {
tc.t.Errorf("FetchIdentityByAddress (%s): got error %v", tc.dbType,
err)
}
if !reflect.DeepEqual(&idV4.Address, addrV4) {
tc.t.Errorf("FetchIdentityByAddress (%s): identities not equal",
tc.dbType)
}

// test RemovePubKey for an address that exists in the database
// Test whether RemoveEncryptedPubKey fails for a decrypted key.
tag, _ = wire.NewShaHash(idV4.Tag())
err = tc.db.RemoveEncryptedPubKey(tag)
if err != database.ErrNonexistentObject {
tc.t.Errorf("RemoveEncryptedPubKey (%s): expected nonexistent object"+
" error, got %v", tc.dbType, err)
}

// Test RemoveEncryptedPubKey for a tag that exists in the database
data, _ = hex.DecodeString("0000000000A229F300000000559BAFB0000000010401F17457B60376F21C20F485652A8A3047ECBBAD4E609E3594D74C7A94D55B994BB4385338FF5E76639EA4EB083C01CCE402CA00205BA2AE2C4519376892C8352EE978E91B298071FFC2F9A211B566715C1D8A4EFF002018E8708FACA508F037DB07E05015B3F91BAAAE22F6A5FF24C4F0C9AF3B1119B01AE14110E14ECC4B511D867DDA589F26A257AD07244F1BCA502C2BFF4269AF29F99C2437C555F4CC42FF510607E4787082354D6C4CDD437010EFDC9B7C88783855988578890C5CBB1FBAE692B7B40C6E1B4642299E181BADCB48D9F931701BA5351161C7E136703DDE5D85735F6366184195C68EF0FB401304B068ED10E785B78C503A3D15FE5D302B0D660E611004BC2925D45D4F65F6E9607F5EA5CA64CBF87B753318AC55364D471B589FFD946E85AC8BA3B6FE82E71A34403852C55A4F4CDCECB89F77032AB8BF18D7C68A983CB902456C90367D3494FA040A69830F4A140E251BCD78A7A36E5F674D5B91C553EF2A3DA17D0F1E9A05F154719F39CBEEF1")
msg = new(wire.MsgObject)
err = msg.Decode(bytes.NewReader(data))
if err != nil {
tc.t.Fatal("failed to decode v4 pubkey, got error", err)
}
_, err = tc.db.InsertObject(msg)
if err != nil {
tc.t.Errorf("InsertObject (%s): got error %v", tc.dbType, err)
}
tag, _ = wire.NewShaHash(msg.Payload[:32])
err = tc.db.RemoveEncryptedPubKey(tag)
if err != nil {
tc.t.Errorf("RemoveEncryptedPubKey (%s): got error %v", tc.dbType, err)
}

// test RemovePublicIdentity for an address that exists in the database
// v2
err = tc.db.RemovePublicIdentity(addrV2)
if err != nil {
tc.t.Errorf("RemovePublicIdentity (%s): got error %v", tc.dbType, err)
}
// v3
err = tc.db.RemovePublicIdentity(addrV3)
if err != nil {
tc.t.Errorf("RemovePublicIdentity (%s): got error %v", tc.dbType, err)
}
// decrypted v4
err = tc.db.RemovePublicIdentity(addrV4)
if err != nil {
tc.t.Errorf("RemovePublicIdentity (%s): got error %v", tc.dbType, err)
}
// Test RemovePubKey for an address that was removed.
err = tc.db.RemovePublicIdentity(addrV2)
if err != database.ErrNonexistentObject {
tc.t.Errorf("RemovePublicIdentity (%s): expected nonexistent object"+
" error, got %v", tc.dbType, err)
}

// test FetchIdentityByAddress for an address that was removed
// v2 address
_, err = tc.db.FetchIdentityByAddress(addrV2)
if err != database.ErrNonexistentObject {
tc.t.Errorf("FetchIdentityByAddress (%s): expected nonexistent object"+
" error, got %v", tc.dbType, err)
}
// v3 address
_, err = tc.db.FetchIdentityByAddress(addrV3)
if err != database.ErrNonexistentObject {
tc.t.Errorf("FetchIdentityByAddress (%s): expected nonexistent object"+
" error, got %v", tc.dbType, err)
}
// v4 address

// test RemovePubKey for an address that was removed
_, err = tc.db.FetchIdentityByAddress(addrV4)
if err != database.ErrNonexistentObject {
tc.t.Errorf("FetchIdentityByAddress (%s): expected nonexistent object"+
" error, got %v", tc.dbType, err)
}
}

// tests FetchRandomInvHashes and FilterObjects
Expand Down

0 comments on commit ffbccea

Please sign in to comment.