From e58a139d145494f7012e7d980d2f257fb4a1e71e Mon Sep 17 00:00:00 2001 From: JoeGruffins <34998433+JoeGruffins@users.noreply.github.com> Date: Sat, 12 Jun 2021 04:05:41 +0900 Subject: [PATCH] client/db: add archivedOrdersBucket In order to make searching for active orders in the database faster, split the order bucket into active and archived orders. Add order.OrderStatus.IsActive method. --- client/db/bolt/db.go | 319 ++++++++++++++++++++++--------- client/db/bolt/db_test.go | 7 +- client/db/bolt/testdata/v3.db.gz | Bin 0 -> 9302 bytes client/db/bolt/upgrades.go | 61 ++++++ client/db/bolt/upgrades_test.go | 69 ++++++- dex/order/status.go | 5 + 6 files changed, 365 insertions(+), 96 deletions(-) create mode 100644 client/db/bolt/testdata/v3.db.gz diff --git a/client/db/bolt/db.go b/client/db/bolt/db.go index 4076aa17bd..dfd8ca9cbe 100644 --- a/client/db/bolt/db.go +++ b/client/db/bolt/db.go @@ -52,7 +52,8 @@ var ( appBucket = []byte("appBucket") accountsBucket = []byte("accounts") disabledAccountsBucket = []byte("disabledAccounts") - ordersBucket = []byte("orders") + activeOrdersBucket = []byte("activeOrders") + archivedOrdersBucket = []byte("orders") matchesBucket = []byte("matches") walletsBucket = []byte("wallets") notesBucket = []byte("notes") @@ -116,7 +117,7 @@ func NewDB(dbPath string, logger dex.Logger) (dexdb.DB, error) { } err = bdb.makeTopLevelBuckets([][]byte{appBucket, accountsBucket, disabledAccountsBucket, - ordersBucket, matchesBucket, walletsBucket, notesBucket}) + activeOrdersBucket, archivedOrdersBucket, matchesBucket, walletsBucket, notesBucket}) if err != nil { return nil, err } @@ -431,18 +432,43 @@ func (db *BoltDB) disabledAcctsUpdate(f bucketFunc) error { // info for the same order ID will be overwritten without indication. func (db *BoltDB) UpdateOrder(m *dexdb.MetaOrder) error { ord, md := m.Order, m.MetaData + if md.Status == order.OrderStatusUnknown { + return fmt.Errorf("cannot set order %s status to unknown", ord.ID()) + } if md.Host == "" { return fmt.Errorf("empty DEX not allowed") } if len(md.Proof.DEXSig) == 0 { return fmt.Errorf("cannot save order without DEX signature") } - return db.ordersUpdate(func(master *bbolt.Bucket) error { + return db.ordersUpdate(func(ob, archivedOB *bbolt.Bucket) error { oid := ord.ID() - oBkt, err := master.CreateBucketIfNotExists(oid[:]) + + // Create or move an order bucket based on order status. Active + // orders go in the activeOrdersBucket. Inactive orders go in the + // archivedOrdersBucket. + bkt := ob + whichBkt := "active" + inactive := !md.Status.IsActive() + if inactive { + // No order means that it was never added to the active + // orders bucket, or UpdateOrder was already called on + // this order after it became inactive. + if ob.Bucket(oid[:]) != nil { + // This order now belongs in the archived orders + // bucket. Delete it from active orders. + if err := ob.DeleteBucket(oid[:]); err != nil { + return fmt.Errorf("archived order bucket delete error: %w", err) + } + } + bkt = archivedOB + whichBkt = "archived" + } + oBkt, err := bkt.CreateBucketIfNotExists(oid[:]) if err != nil { - return fmt.Errorf("order bucket error: %w", err) + return fmt.Errorf("%s bucket error: %w", whichBkt, err) } + var linkedB []byte if !md.LinkedOrder.IsZero() { linkedB = md.LinkedOrder[:] @@ -469,10 +495,9 @@ func (db *BoltDB) UpdateOrder(m *dexdb.MetaOrder) error { // ActiveOrders retrieves all orders which appear to be in an active state, // which is either in the epoch queue or in the order book. func (db *BoltDB) ActiveOrders() ([]*dexdb.MetaOrder, error) { - return db.filteredOrders(func(oBkt *bbolt.Bucket) bool { - status := oBkt.Get(statusKey) - return bEqual(status, byteEpoch) || bEqual(status, byteBooked) - }) + return db.filteredOrders(func(_ *bbolt.Bucket) bool { + return true + }, false) } // AccountOrders retrieves all orders associated with the specified DEX. n = 0 @@ -483,13 +508,13 @@ func (db *BoltDB) AccountOrders(dex string, n int, since uint64) ([]*dexdb.MetaO if n == 0 && since == 0 { return db.filteredOrders(func(oBkt *bbolt.Bucket) bool { return bEqual(dexB, oBkt.Get(dexKey)) - }) + }, true) } sinceB := uint64Bytes(since) return db.newestOrders(n, func(_ []byte, oBkt *bbolt.Bucket) bool { timeB := oBkt.Get(updateTimeKey) return bEqual(dexB, oBkt.Get(dexKey)) && bytes.Compare(timeB, sinceB) >= 0 - }) + }, true) } // MarketOrders retrieves all orders for the specified DEX and market. n = 0 @@ -509,9 +534,8 @@ func (db *BoltDB) MarketOrders(dex string, base, quote uint32, n int, since uint func (db *BoltDB) ActiveDEXOrders(dex string) ([]*dexdb.MetaOrder, error) { dexB := []byte(dex) return db.filteredOrders(func(oBkt *bbolt.Bucket) bool { - status := oBkt.Get(statusKey) - return bEqual(dexB, oBkt.Get(dexKey)) && (bEqual(status, byteEpoch) || bEqual(status, byteBooked)) - }) + return bEqual(dexB, oBkt.Get(dexKey)) + }, false) } // marketOrdersAll retrieves all orders for the specified DEX and market. @@ -519,7 +543,7 @@ func (db *BoltDB) marketOrdersAll(dexB, baseB, quoteB []byte) ([]*dexdb.MetaOrde return db.filteredOrders(func(oBkt *bbolt.Bucket) bool { return bEqual(dexB, oBkt.Get(dexKey)) && bEqual(baseB, oBkt.Get(baseKey)) && bEqual(quoteB, oBkt.Get(quoteKey)) - }) + }, true) } // marketOrdersSince grabs market orders with optional filters for maximum count @@ -532,20 +556,23 @@ func (db *BoltDB) marketOrdersSince(dexB, baseB, quoteB []byte, n int, since uin timeB := oBkt.Get(updateTimeKey) return bEqual(dexB, oBkt.Get(dexKey)) && bEqual(baseB, oBkt.Get(baseKey)) && bEqual(quoteB, oBkt.Get(quoteKey)) && bytes.Compare(timeB, sinceB) >= 0 - }) + }, true) } // newestOrders returns the n newest orders, filtered with a supplied filter // function. Each order's bucket is provided to the filter, and a boolean true // return value indicates the order should is eligible to be decoded and // returned. -func (db *BoltDB) newestOrders(n int, filter func([]byte, *bbolt.Bucket) bool) ([]*dexdb.MetaOrder, error) { - var orders []*dexdb.MetaOrder - return orders, db.ordersView(func(master *bbolt.Bucket) error { - pairs := newestBuckets(master, n, updateTimeKey, filter) - for _, pair := range pairs { - oBkt := master.Bucket(pair.k) - o, err := decodeOrderBucket(pair.k, oBkt) +func (db *BoltDB) newestOrders(n int, filter func([]byte, *bbolt.Bucket) bool, includeArchived bool) ([]*dexdb.MetaOrder, error) { + orders := make([]*dexdb.MetaOrder, 0, n) + return orders, db.ordersView(func(ob, archivedOB *bbolt.Bucket) error { + buckets := []*bbolt.Bucket{ob} + if includeArchived { + buckets = append(buckets, archivedOB) + } + trios := newestBuckets(buckets, n, updateTimeKey, filter) + for _, trio := range trios { + o, err := decodeOrderBucket(trio.k, trio.b) if err != nil { return err } @@ -558,31 +585,46 @@ func (db *BoltDB) newestOrders(n int, filter func([]byte, *bbolt.Bucket) bool) ( // filteredOrders gets all orders that pass the provided filter function. Each // order's bucket is provided to the filter, and a boolean true return value // indicates the order should be decoded and returned. -func (db *BoltDB) filteredOrders(filter func(*bbolt.Bucket) bool) ([]*dexdb.MetaOrder, error) { +func (db *BoltDB) filteredOrders(filter func(*bbolt.Bucket) bool, includeArchived bool) ([]*dexdb.MetaOrder, error) { var orders []*dexdb.MetaOrder - return orders, db.ordersView(func(master *bbolt.Bucket) error { - return master.ForEach(func(oid, _ []byte) error { - oBkt := master.Bucket(oid) - if oBkt == nil { - return fmt.Errorf("order %x bucket is not a bucket", oid) - } - if filter(oBkt) { - o, err := decodeOrderBucket(oid, oBkt) - if err != nil { - return err + return orders, db.ordersView(func(ob, archivedOB *bbolt.Bucket) error { + buckets := []*bbolt.Bucket{ob} + if includeArchived { + buckets = append(buckets, archivedOB) + } + for _, master := range buckets { + err := master.ForEach(func(oid, _ []byte) error { + oBkt := master.Bucket(oid) + if oBkt == nil { + return fmt.Errorf("order %x bucket is not a bucket", oid) } - orders = append(orders, o) + if filter(oBkt) { + o, err := decodeOrderBucket(oid, oBkt) + if err != nil { + return err + } + orders = append(orders, o) + } + return nil + }) + if err != nil { + return err } - return nil - }) + } + return nil }) } // Order fetches a MetaOrder by order ID. func (db *BoltDB) Order(oid order.OrderID) (mord *dexdb.MetaOrder, err error) { oidB := oid[:] - err = db.ordersView(func(master *bbolt.Bucket) error { - oBkt := master.Bucket(oidB) + err = db.ordersView(func(ob, archivedOB *bbolt.Bucket) error { + oBkt := ob.Bucket(oidB) + // If the order is not in the active bucket, check the archived + // orders bucket. + if oBkt == nil { + oBkt = archivedOB.Bucket(oidB) + } if oBkt == nil { return fmt.Errorf("order %s not found", oid) } @@ -645,6 +687,7 @@ func (db *BoltDB) Orders(orderFilter *db.OrderFilter) (ords []*dexdb.MetaOrder, }) } + includeArchived := true if len(orderFilter.Statuses) > 0 { filters = append(filters, func(_ []byte, oBkt *bbolt.Bucket) bool { status := order.OrderStatus(intCoder.Uint16(oBkt.Get(statusKey))) @@ -655,14 +698,25 @@ func (db *BoltDB) Orders(orderFilter *db.OrderFilter) (ords []*dexdb.MetaOrder, } return false }) + includeArchived = false + for _, status := range orderFilter.Statuses { + if !status.IsActive() { + includeArchived = true + break + } + } } if !orderFilter.Offset.IsZero() { - offsetOID := orderFilter.Offset[:] - var offsetBucket *bbolt.Bucket + offsetOID := orderFilter.Offset var stampB []byte - err := db.ordersView(func(master *bbolt.Bucket) error { - offsetBucket = master.Bucket(offsetOID) + err := db.ordersView(func(ob, archivedOB *bbolt.Bucket) error { + offsetBucket := ob.Bucket(offsetOID[:]) + // If the order is not in the active bucket, check the + // archived orders bucket. + if offsetBucket == nil { + offsetBucket = archivedOB.Bucket(offsetOID[:]) + } if offsetBucket == nil { return fmt.Errorf("order %s not found", offsetOID) } @@ -675,11 +729,11 @@ func (db *BoltDB) Orders(orderFilter *db.OrderFilter) (ords []*dexdb.MetaOrder, filters = append(filters, func(oidB []byte, oBkt *bbolt.Bucket) bool { comp := bytes.Compare(oBkt.Get(updateTimeKey), stampB) - return comp < 0 || (comp == 0 && bytes.Compare(offsetOID, oidB) < 0) + return comp < 0 || (comp == 0 && bytes.Compare(offsetOID[:], oidB) < 0) }) } - return db.newestOrders(orderFilter.N, filters.check) + return db.newestOrders(orderFilter.N, filters.check, includeArchived) } // decodeOrderBucket decodes the order's *bbolt.Bucket into a *MetaOrder. @@ -729,12 +783,56 @@ func decodeOrderBucket(oid []byte, oBkt *bbolt.Bucket) (*dexdb.MetaOrder, error) }, nil } +// updateOrderBucket expects an oid to exist in either the orders or archived +// order buckets. If status is not active, it first checks active orders. If +// found it moves the order to the archived bucket and returns that order +// bucket. If status is less than or equal to booked, it expects the order +// to already be in active orders and returns that bucket. +func updateOrderBucket(ob, archivedOB *bbolt.Bucket, oid order.OrderID, status order.OrderStatus) (*bbolt.Bucket, error) { + if status == order.OrderStatusUnknown { + return nil, fmt.Errorf("cannot set order %s status to unknown", oid) + } + inactive := !status.IsActive() + if inactive { + if bkt := ob.Bucket(oid[:]); bkt != nil { + // It's in the active bucket. Move it to the archived bucket. + oBkt, err := archivedOB.CreateBucket(oid[:]) + if err != nil { + return nil, fmt.Errorf("unable to create archived order bucket: %v", err) + } + // Assume the order bucket contains only values, no + // sub-buckets + if err := bkt.ForEach(func(k, v []byte) error { + return oBkt.Put(k, v) + }); err != nil { + return nil, fmt.Errorf("unable to copy active order bucket: %v", err) + } + if err := ob.DeleteBucket(oid[:]); err != nil { + return nil, fmt.Errorf("unable to delete active order bucket: %v", err) + } + return oBkt, nil + } + // It's not in the active bucket, check archived. + oBkt := archivedOB.Bucket(oid[:]) + if oBkt == nil { + return nil, fmt.Errorf("archived order %s not found", oid) + } + return oBkt, nil + } + // Active status should be in the active bucket. + oBkt := ob.Bucket(oid[:]) + if oBkt == nil { + return nil, fmt.Errorf("active order %s not found", oid) + } + return oBkt, nil +} + // UpdateOrderMetaData updates the order metadata, not including the Host. func (db *BoltDB) UpdateOrderMetaData(oid order.OrderID, md *db.OrderMetaData) error { - return db.ordersUpdate(func(master *bbolt.Bucket) error { - oBkt := master.Bucket(oid[:]) - if oBkt == nil { - return fmt.Errorf("UpdateOrderMetaData - order %s not found", oid) + return db.ordersUpdate(func(ob, archivedOB *bbolt.Bucket) error { + oBkt, err := updateOrderBucket(ob, archivedOB, oid, md.Status) + if err != nil { + return fmt.Errorf("UpdateOrderMetaData: %w", err) } var linkedB []byte @@ -757,10 +855,10 @@ func (db *BoltDB) UpdateOrderMetaData(oid order.OrderID, md *db.OrderMetaData) e // UpdateOrderStatus sets the order status for an order. func (db *BoltDB) UpdateOrderStatus(oid order.OrderID, status order.OrderStatus) error { - return db.ordersUpdate(func(master *bbolt.Bucket) error { - oBkt := master.Bucket(oid[:]) - if oBkt == nil { - return fmt.Errorf("UpdateOrderStatus - order %s not found", oid) + return db.ordersUpdate(func(ob, archivedOB *bbolt.Bucket) error { + oBkt, err := updateOrderBucket(ob, archivedOB, oid, status) + if err != nil { + return fmt.Errorf("UpdateOrderStatus: %w", err) } return oBkt.Put(statusKey, uint16Bytes(uint16(status))) }) @@ -768,8 +866,13 @@ func (db *BoltDB) UpdateOrderStatus(oid order.OrderID, status order.OrderStatus) // LinkOrder sets the linked order. func (db *BoltDB) LinkOrder(oid, linkedID order.OrderID) error { - return db.ordersUpdate(func(master *bbolt.Bucket) error { - oBkt := master.Bucket(oid[:]) + return db.ordersUpdate(func(ob, archivedOB *bbolt.Bucket) error { + oBkt := ob.Bucket(oid[:]) + // If the order is not in the active bucket, check the archived + // orders bucket. + if oBkt == nil { + oBkt = archivedOB.Bucket(oid[:]) + } if oBkt == nil { return fmt.Errorf("LinkOrder - order %s not found", oid) } @@ -781,14 +884,40 @@ func (db *BoltDB) LinkOrder(oid, linkedID order.OrderID) error { }) } -// ordersView is a convenience function for reading from the order bucket. -func (db *BoltDB) ordersView(f bucketFunc) error { - return db.withBucket(ordersBucket, db.View, f) +// ordersView is a convenience function for reading from the order buckets. +// Orders are spread over two buckets to make searching active orders faster. +// Any reads of the order buckets should be done in the same transaction, as +// orders may move from active to archived at any time. +func (db *BoltDB) ordersView(f func(ob, archivedOB *bbolt.Bucket) error) error { + return db.View(func(tx *bbolt.Tx) error { + ob := tx.Bucket(activeOrdersBucket) + if ob == nil { + return fmt.Errorf("failed to open %s bucket", string(activeOrdersBucket)) + } + archivedOB := tx.Bucket(archivedOrdersBucket) + if archivedOB == nil { + return fmt.Errorf("failed to open %s bucket", string(archivedOrdersBucket)) + } + return f(ob, archivedOB) + }) } -// ordersUpdate is a convenience function for updating the order bucket. -func (db *BoltDB) ordersUpdate(f bucketFunc) error { - return db.withBucket(ordersBucket, db.Update, f) +// ordersUpdate is a convenience function for updating the order buckets. +// Orders are spread over two buckets to make searching active orders faster. +// Any writes of the order buckets should be done in the same transaction to +// ensure that reads can be kept concurrent. +func (db *BoltDB) ordersUpdate(f func(ob, archivedOB *bbolt.Bucket) error) error { + return db.Update(func(tx *bbolt.Tx) error { + ob := tx.Bucket(activeOrdersBucket) + if ob == nil { + return fmt.Errorf("failed to open %s bucket", string(activeOrdersBucket)) + } + archivedOB := tx.Bucket(archivedOrdersBucket) + if archivedOB == nil { + return fmt.Errorf("failed to open %s bucket", string(archivedOrdersBucket)) + } + return f(ob, archivedOB) + }) } // UpdateMatch updates the match information in the database. Any existing @@ -1140,14 +1269,13 @@ func (db *BoltDB) AckNotification(id []byte) error { func (db *BoltDB) NotificationsN(n int) ([]*dexdb.Notification, error) { notes := make([]*dexdb.Notification, 0, n) return notes, db.notesView(func(master *bbolt.Bucket) error { - pairs := newestBuckets(master, n, stampKey, nil) - for _, pair := range pairs { - noteBkt := master.Bucket(pair.k) - note, err := dexdb.DecodeNotification(getCopy(noteBkt, noteKey)) + trios := newestBuckets([]*bbolt.Bucket{master}, n, stampKey, nil) + for _, trio := range trios { + note, err := dexdb.DecodeNotification(getCopy(trio.b, noteKey)) if err != nil { return err } - note.Ack = bEqual(noteBkt.Get(ackKey), byteTrue) + note.Ack = bEqual(trio.b.Get(ackKey), byteTrue) note.Id = note.ID() notes = append(notes, note) } @@ -1165,20 +1293,22 @@ func (db *BoltDB) notesUpdate(f bucketFunc) error { return db.withBucket(notesBucket, db.Update, f) } -// Newest buckets gets the nested buckets with the hightest timestamp from the -// specified master bucket. The nested bucket should have an encoded uint64 at +// newest buckets gets the nested buckets with the hightest timestamp from the +// specified master buckets. The nested bucket should have an encoded uint64 at // the timeKey. An optional filter function can be used to reject buckets. -func newestBuckets(master *bbolt.Bucket, n int, timeKey []byte, filter func([]byte, *bbolt.Bucket) bool) []*keyTimePair { +func newestBuckets(buckets []*bbolt.Bucket, n int, timeKey []byte, filter func([]byte, *bbolt.Bucket) bool) []*keyTimeTrio { idx := newTimeIndexNewest(n) - master.ForEach(func(k, _ []byte) error { - bkt := master.Bucket(k) - stamp := intCoder.Uint64(bkt.Get(timeKey)) - if filter == nil || filter(k, bkt) { - idx.add(stamp, k) - } - return nil - }) - return idx.pairs + for _, master := range buckets { + master.ForEach(func(k, _ []byte) error { + bkt := master.Bucket(k) + stamp := intCoder.Uint64(bkt.Get(timeKey)) + if filter == nil || filter(k, bkt) { + idx.add(stamp, k, bkt) + } + return nil + }) + } + return idx.trios } // makeTopLevelBuckets creates a top-level bucket for each of the provided keys, @@ -1264,55 +1394,58 @@ func (bp *bucketPutter) err() error { return bp.putErr } -// keyTimePair is used to build an on-the-fly time-sorted index. -type keyTimePair struct { +// keyTimeTrio is used to build an on-the-fly time-sorted index. +type keyTimeTrio struct { k []byte t uint64 + b *bbolt.Bucket } -// timeIndexNewest is a struct used to build an index of sorted keyTimePairs. +// timeIndexNewest is a struct used to build an index of sorted keyTimeTrios. // The index can have a maximum capacity. If the capacity is set to zero, the // index size is unlimited. type timeIndexNewest struct { - pairs []*keyTimePair + trios []*keyTimeTrio cap int } // Create a new *timeIndexNewest, with the specified capacity. func newTimeIndexNewest(n int) *timeIndexNewest { return &timeIndexNewest{ - pairs: make([]*keyTimePair, 0, n), + trios: make([]*keyTimeTrio, 0, n), cap: n, } } -// Conditionally add a time-key pair to the index. The pair will only be added +// Conditionally add a time-key trio to the index. The trio will only be added // if the timeIndexNewest is under capacity and the time t is larger than the -// oldest pair's time. -func (idx *timeIndexNewest) add(t uint64, k []byte) { - count := len(idx.pairs) +// oldest trio's time. +func (idx *timeIndexNewest) add(t uint64, k []byte, b *bbolt.Bucket) { + count := len(idx.trios) if idx.cap == 0 || count < idx.cap { - idx.pairs = append(idx.pairs, &keyTimePair{ + idx.trios = append(idx.trios, &keyTimeTrio{ // Need to make a copy, and []byte(k) upsets the linter. k: append([]byte(nil), k...), t: t, + b: b, }) } else { // non-zero length, at capacity. - if t <= idx.pairs[count-1].t { + if t <= idx.trios[count-1].t { // Too old. Discard. return } - idx.pairs[count-1] = &keyTimePair{ + idx.trios[count-1] = &keyTimeTrio{ k: append([]byte(nil), k...), t: t, + b: b, } } - sort.Slice(idx.pairs, func(i, j int) bool { + sort.Slice(idx.trios, func(i, j int) bool { // newest (highest t) first, or by lexicographically by key if the times // are equal. - t1, t2 := idx.pairs[i].t, idx.pairs[j].t - return t1 > t2 || (t1 == t2 && bytes.Compare(idx.pairs[i].k, idx.pairs[j].k) == 1) + t1, t2 := idx.trios[i].t, idx.trios[j].t + return t1 > t2 || (t1 == t2 && bytes.Compare(idx.trios[i].k, idx.trios[j].k) == 1) }) } diff --git a/client/db/bolt/db_test.go b/client/db/bolt/db_test.go index 81083b9bda..79fd87cbab 100644 --- a/client/db/bolt/db_test.go +++ b/client/db/bolt/db_test.go @@ -664,8 +664,11 @@ func TestOrderFilters(t *testing.T) { } // Set the update time. - boltdb.ordersUpdate(func(master *bbolt.Bucket) error { - oBkt := master.Bucket(oid[:]) + boltdb.ordersUpdate(func(aob, eob *bbolt.Bucket) error { + oBkt := aob.Bucket(oid[:]) + if oBkt == nil { + oBkt = eob.Bucket(oid[:]) + } if oBkt == nil { t.Fatalf("order %s not found", oid) } diff --git a/client/db/bolt/testdata/v3.db.gz b/client/db/bolt/testdata/v3.db.gz new file mode 100644 index 0000000000000000000000000000000000000000..ddf9c77276ba800f5989b1db6832bf84660b1cb5 GIT binary patch literal 9302 zcmcJSg;N|nyY{i-?i4RpoMOeTXtCn%QruazECq^7f#U8C#hu07DZ0QG7I*h$zh}-n z&-u>z1I}+I$>dJvo=kGhm4r419sScoqB#OGfRD@C65%8p&6sHQZX_8WzsMZ?#HJnpXtfV^?pa&*S|8>gW$HMIFuXwD`8%E7WGpH}LPQ&}Id zf)P!$NIXSw@b~}4BM@zVBS;PYEO2@6(;D{#4Cy41`{f^wCp_|!N5O>sznFv|-u4W2 zNwyMiRig1Pq3o?n(nR6~AV#0kst-mq1A&Z4bN!o^8vr=7AobzZNCz^exCy{2um^kx z07n4^ur3P(`(6(GKhtTV{XGStYoM?te%6eMOEN}DZEA2rW^lW-R;tKs zFS?EYDWQd8Aha{*)2MmBq5oFkl?ia3(GJt~Kk1Z2Ntnqdd-+oa?0@>G;brW^IyCW4 z{Z9@c-V!?IwCpC9XHs{Tf+;O#(;!PqL0eF4ME5pMy)9$4eX;?+-D;um18+?i~Xz}BztoGtFBk{Dz$DX|Kx)~4NW zFDzGDMzT;WV0gZ2PD*V(4aJoL5WDlOn1XtI;kzM^=>k%#3u7S1`}}^e?|}cMPSsvx-lSBJnY<&7%enn}|Tric8u>Q6B(j;pCcp%556WE^qw^Ld;d1Ag^M@Gx=y`^PnJ~y@(-BvYNJRTR|VH*^PLMGcHN6GCgmOU9XCX} zTv(ul<<4M9VM-ME{7XMzzAM1yk>g0mMoxlv7V5vm?-aqx9U5l@tqVxaJxS3E;I)cz zgT_}bDj!ChIUG|J&1N;pkHiolQVcnmKtruR#xkQj^X^sHoMpH?KIXmQ`4{-3ZyOk> z0H1LKBdXp=v`0;OAH(r|q-=;_+|GsYA=MkuMco0M?)a12>KaaHaU@tPH zo_!lJwKx1_MA2?l0zimFEot|DdaiuHyUe`{2fK{Y?IS`|@xra9Ht8uuG-aPq? zXO$o?t1rmq99?VWnWi|Ro?oKQ6&m}{2a~37a)uTw#JKP#f@(Wt%lQkw_Hd)TJe2w8 zJN|gXc`h>)wi=Pw`ps?jP7&-#8F@B8ZGs+oYPjcUv;ceVM*DCoIutSxz#&cLL5$Qx z{5If8br#D|8CL3D&%)W6&~!r%TrFFyRgp=U`Z|&4&)viK!H%&-CWJzQ`GDD~Hh*Ls_|pKItGp6n$5Sn7uE5LRba*}qw?dh#Ukjfd-r?B1E0=C7gIabyT;BR+XBDeQ zxkAM7cIlAc?y0RZ-Go)XQ0#fMUErpmq^~U8{0^H5a*1I~06R;%Rzp2ae_XsVuN%4* zfiN62xg6+oz*hzyY;d%;HC=kzf`og@7TBlhA2o{-OhCHbcYP1kWM{`_{Ico4Y64uG zv%Cv-@p9QC1(tN*SGOHk47M^^MCw6?|46PIL>-F=@3o!{C5&1BJMLYY*l+q;T-nC6xcxye71?~JSZ}XA0Mo$>)|(_7o_MnygexYAWX$g*?2#7q zhyJg-*RLY%XfIs^{D_z$6JUU7%^juSsrEN7NJt9HXIY}gX@?}G$4&Y@V;Yka z^@r6P%gS8?%|h<7^Lz^rF&7n%1g$Si4{UNz0HScgo)M}`lIaGigs;P2hvo~gh6Z6a z#kroL#{ni67X@DD%gFA6*UT3jq(5xbF6*9z?*v(MJ^DM3yOq=@2toa~E()xJO@~9A z)8`-FP#j#6Lq8h)g`51HhW?@N&hzPz<4mIeUD-_SJ^}(so0$PG@IE8meLeDFLVWIG zF#nfVBfCF;;grYlc<(?ohAxRkxNV*Kt>n&a#jBn#>D4+2^a?M@U2lLZGrW4i;7iDV-w0-9#hbOxcbv=@p@lOc+KWNvgOFigLU0-FANH5w z$8VS6i+V13xKa%t@zzaYnf}$~PHGTuCfXV}JcFhqw>R$I$&7|Z>^w*9K`c-!qHT&9 z-(<7x)zwc})Ni1lsjcE~Tujb3vsumP!;xde30%|eq=bLxhLfY%JFK%Q~`Oy2VK6D5cj62JA z+1y;5@1K9#_~qG*SRSn z2FbZT+Uo6j&p}Q*>*ly{{rR<&CECub^(AS;1PmyR;kzj~CH^`**>PlQoF~}f2R+Rf zxPDj?aP1N|$`syw7jXiNa5y)7x!32tAI^2(Y&`dk+xU2rB_N>%Xb$LhxjM3{|A1cz zjtlUK&TgHhpQ3$Z0@8kcS^Gm1(C&?w9=>dZdWe>w`6fb7Auo9n1R#I~r3LG~qPTc$ zaX?dhN6nrJ)=Z?ys17|h@!vG846RmSd6IT)lphZJTxLG?yq7gh5KEg zRV<+Du>+6a%S+Nrw=LJNLHht5yxOXjbGOsT3kJ17V^RA29v{W82TQsM0qP3L0({0` zj3=sf$kDH_WBqEEO1G}t&|{XxxD5YS!0zNH$0~|sBuBaT)m;{lBg678r!wgnRlwEJ9uJ#c4MWd78mJE}ucWW%t`1ITRWq)=)cQT@6n&1Pp8ih6*Nu zX8H}nY1q~8rTKPx_Se@=KiP$-0ANQIX|a+PP4?wca;ZA#%+P&coAj>$k^z5JX%LpG=(aRaP&)9af}DxF8R& zeaCA)vn2VTgn|4YuJgVdC4p~2$7V=EJv){JTs7aOoh3cM;M2UYwt?32#ILPDE}A1) z{6BB-1ULJ@W^nFq{6hJ08J)y|e;hQ6LY^!6a8R&W#p;BZCD~$QWJL^Ti7P}p6g$L{ zchGNG3@0+h5l0_Mw{UUv8XM4^Q?@9zLX`ol#&+b7EWstKCjb}A2j93wh5e?e|2fIQ zw?pAfL)(M3HY0H;>2%=zeWn|fWSRsH&nzx`B0FqGVgqv{%({ zUI|~cRoxCQ>|oqU?Xp+fy1?13n`eZcRXsJELnRyF}(^5Th}lNRm{XN1R2 zx4d9~I0&oF0W)44@!JNVy3-Z8zrM7N(29%NJ{%YCma7tBZz*s47ufNzQ-Rq z{tvVyQt!ryE1}V{ws~mAwYr=GtWCVx+O`7YbPQ^2AN~%qnbeYhy{Gu=)*or2lS*Ep z!X;P{OFnWU#4;tUij1G5l^5a;sm#@0>Kfa#94Bn+t*Nm;@l4PZr9#o#H#1#gj=6sF zr48fiEh`_&Hm&%{98u9SzwqNHA@LyAltQ@u9iba4|!Qq2046i_z;#TlxzaHiZ=}NOy5r#~!zoLNRk0&K>f61kIPc3$ zRj=YR(ZH&3IiurrzTv0M`&h9rA5PEe#UXm&4zG6`MkvPVd<)_@O=PVp9?>j&0J#JDj_7R$p)Vnoq+RWuZpeFN--snTN< zFA;=j>E8ZQM7pfJv1B8^@jqggZ7E;T%NBHQ7{lpl74yI6OXyW7e%`yL;75%cP_{AR zagrKT+`lGD*G0%oHI*Xx7?R=18SMS5xXC)lE z^L(uJBxEnWWK5`NY9K*SL?&9W57zA+SJY7a!^%dHlgds~n$8z>$o%}?qomr4DJ!Ks z$?icHAn`NC9fp`2T%t6SZ2R+aURy>&4_|>e(GI0HxQJkvAfM{l+U-qn%82dmg&jhS z?Lwob-&P{RsI`S%1@l`R1Y#+nm^TQ4QY#a3}IBGQD}As1hPbNWFb!r$$hx5=r& zXx2&ChVLY_l#AV=QzFVCnmAJ0NaDnwO6%WHF%>5?W|ELna?>j)Z*Pw-MQT$i;8>IM zHuF-)vF~d^nbxeu$$kd=7Np{M5G%E`seB=<>N(PX&EmlOEQlHgqic_m(jw6jU$KzM`jLrCJu8tOa5XeZvBniZ={>4l zr{77Lrt|Cl{j^~7FiHxE(SO%R{xN*&n#*cbpGD z(Qo0K8Xcx^hzz-_ui9hyF+7E2EDPGqVVH2^$YXTMdv&Pz8ESrae7VVVQ?FdA0^NIw zdxGm{2AdYsCa7US(T?T@9w&RuKqm>di|Tgk1a7J-ypu-m0Bkxui}+@ddD zhRE8MM2%k+lC#5E3o733)T4(EL9T;o#2>DVadq*zI<<EArYdsUz$fh7-LO?80 z#!o};3f=$w7g?_+tV~C5TfV@{K;-{9%I$D&G`V4a@ButJvb-)~lTc(8etlc>pTU3X zzaR>UV+h$boN47nGJ^*Wkxz#MmH#R9OOJ5ZI(Whq{8}asd6lYG=~pgyLIv63MCoJZ z!2&rd$M-zMoarq7C>ru%3!w8WC=ClS5c~qaXJ3HB(%wESbE43lJp!voK@TnWoxZJl z=JcDVDU3cja3D)IT>WCSF3V0nUf(G7?d;6KNpH?#N-G0iTw6!uux%!!+dtkd+LRv- z($H2k6M~<6Y_{|s<%cM{L%&2pkJ!rmZ@&70heRL(xeRv6t*)ufVvpa#(}7VecVD2qiplXZ<(Z zGHzk;QQP7s%rf-`?S41PR4mKOPsq%qrrB{t?D_`R*|2xpy{j zE+}FDZeMisp?8zN#UnT+$x)EqD+nAeRi#->`OW>scyO%~V+1eC?Ik}4HVuL2{<(ed zQ$6y3>0}^z6eo!S_4?-8c z7>p1;@@^uWkK2?LKNFf4I8w)_QCLPep=$P1TrM}0jeG(Z*J=P$@NbtfG|c28pV(yn z3!Y)MVwTHlz>-VcvOAeKs$0Q9y2f)S=-Ieq9zRD-2Kls8_O6XqvZ-3ud=PX-AS+LL?6kqgIUr z`)N4r;v1x{tIo1m5f`(O%tY_r(&vs)`aL~2G}Pz-YA-e@t+nX%cuc>Q#H0yUY(qu% zlbcyfl?@YJZif822pw67-aR4H7aE}xY@nv3-=y#Oik4i6jKN_bsnO>&pDod~0JKscf z_m?cHZV+%!2`6c(rgdhJ)$G$nH3;vaYuOr2wQc}!?I>kf8Sarr0V;dvot>$-n>k1U;jZouTRNLG}H(4 zYfj0)Pk8dqZ=-TJ-YtB{1P(D^FISyyewDOY`_A{k?uUB>NB)lyOF{Y~)<#!;Q>>NT zOUrhP$yHM;+m+$3v7n!Jh6z5=Zl>}=oZ4Pw93G3%I?k1n-^e!~OuLbi#tbL3tqVeb{~;MQ_cr zX;LK2EyNe&|fIM3pRs9z~+K)||MOO|EKF^35 zS=*EA)h`5~qE9;L@m1}qQrBH_6@JC`c@H;-ZIfGNDFFW$L+zs)k@1WWTbHL4)!OmV z0vmAWQ8i|ew&;0x;9jilc=^4JBbp7D%GDany3l;$4c}763?}+Dx@SX~f^Y}8WBTu< zK7bGD}k$943hAl>0?BUrS8ePbQ$P&SmWv~zSjCWe`N-%jwP!D~O`6uV* z;95plWGcP&UuFDS^F&7DzInCC1AvU|M{mlxnDU4~8&WRan#QHiqn#SBXVpuqKS(3S zdKMzsPwv~sKjC$|&aA&=Y2=keKtAN3eN!|0#+cvJ(SLeXEIr2O!A|yEHSFRl+6MEt zEVAD#MCD(MBPCsUVT5JaWKDr*jV2tqDvT3snK@TJi44aHJ3h`&5+5ph6PbPvvB9Ev z(`fM8(}WI{0+w2kkIa6-S4p^9*nezfmh*T}DMg%+QW%L$kA|0ikUQC`+cJFIf9bq( zRh19yI-K`l@}CINZvP=XT0aF-?<_y$q+0hRWubp5!Rk>Qf@#ln|qxZD#f}LN^tn5}s14!BB zCr?4z`!o3%*Vnh_>MaJIy!122c`RIxDO7FlCEizzjX#eQf*uSSoYK`TZzjs_v(=_w zIK{W*K`ISz0>clJXL#x@9&+A0glI3dR|$WvnBN&3%~nwQ#F3d`oMp}k;;rbCmh*p* zWqYEqU06c?Nae$Ae^NIlQdves)5+s}taL@xm&mTHs?|Jy^@Ce=d4`7`)t@WAk6!F! zD&Len;Mcb=y^KbsRrNLBQ)=RxStQM$$nq?>>?2`}Jpq^Q$!3It9Ua|mdU}+gHZGg@ z>K=~JT(I&Vj)htcvfaonJ{2c^Bm@MrUI=hBvb;&s|5X9ffF8pO_?0kYGR;$5UD9l;ER zw81?EzsVEh`Dr*K6W+h|m52XLVU!!Y{Ue^}wo$GB5TqdO6UA7f&%E`sQL%64#j?Vx z2@~oN8{7v)3qt|=N27=bt7pyF&T`g&I}J*c4by>>2m197^yR7T1S<8D{Wwnyl#y2&hug4Q_!0UseB&kc7ViIfxfHc(>4moe@-wP6W<_gM>D3cW zy_M@psc8Be%#GT5nbz~lIl-I3=v^kM=yodb3PuQJLcmR7Xr?tM95G8tiXMLi< zY?J;^9>}jP1I$DUhK0{Fqwn!D`i+G|D1^cFCuaV2LH)-G)g9f*K(nb2HM*d3iI(>R zg|g8~8K()H&tGjs5^{&i^qeXi=>>}6B}T;Z1~FXE%L}CyX5J4zmANd9e1P(*jF)2=goUmeD|jjcwOC){Cd`ROiK z3%Dy1E5xf-BQ4}1OinQ>LP7nw=WpS=IZN8uV4}61I@1XXcZ#TW7-bP(sqdCzUm6X& z`w6pnATg=2_oTY1HcU&wbkfsg{Fq$f;xZkVrp^-HbrB?Bx#H@n>-wt23$vU!6kS$3 zD6a8wb7iSe>@wn8gN=v0)9f0v(v0nO>^<1u=j&g$tpy8n1T=h|dh6}Q-P7;~QilFu zSs{6&j`ClEl`{J>>_Sq%Dbz#Li*N#<1BjO6-DCW#WrXb?_t%%NASVSQlqdMV(OwYk zCMe)>l-@*SkD#dZ(_bu?1mRmT%>^Q-UmIvX#_5Saa|I7CE{soPLd!QQu0SV#% E0qn$jCjbBd literal 0 HcmV?d00001 diff --git a/client/db/bolt/upgrades.go b/client/db/bolt/upgrades.go index c66c03f88f..4df5a4c232 100644 --- a/client/db/bolt/upgrades.go +++ b/client/db/bolt/upgrades.go @@ -26,6 +26,8 @@ var upgrades = [...]upgradefunc{ v2Upgrade, // v2 => v3 adds a tx data field to the match proof. v3Upgrade, + // v3 => v4 splits orders into active and archived. + v4Upgrade, } // DBVersion is the latest version of the database that is understood. Databases @@ -126,6 +128,7 @@ func v1Upgrade(dbtx *bbolt.Tx) error { // avoids any chance of rejecting a pre-existing active match. func v2Upgrade(dbtx *bbolt.Tx) error { const oldVersion = 1 + ordersBucket := []byte("orders") dbVersion, err := getVersionTx(dbtx) if err != nil { @@ -232,6 +235,64 @@ func reloadMatchProofs(tx *bbolt.Tx, skipCancels bool) error { }) } +// v4Upgrade moves active orders from what will become the archivedOrdersBucket +// to a new ordersBucket. This is done in order to make searching active orders +// faster, as they do not need to be pulled out of all orders any longer. This +// upgrade moves active orders as opposed to inactive orders under the +// assumption that there are less active orders to move, and so a smaller +// database transaction occurs. +func v4Upgrade(dbtx *bbolt.Tx) error { + const oldVersion = 3 + + if err := ensureVersion(dbtx, oldVersion); err != nil { + return err + } + + // Move any inactive orders to the new archivedOrdersBucket. + return moveActiveOrders(dbtx) +} + +// moveActiveOrders searches the v1 ordersBucket for orders that are inactive, +// adds those to the v2 ordersBucket, and deletes them from the v1 ordersBucket, +// which becomes the archived orders bucket. +func moveActiveOrders(tx *bbolt.Tx) error { + oldOrdersBucket := []byte("orders") + newActiveOrdersBucket := []byte("activeOrders") + // NOTE: newActiveOrdersBucket created in NewDB, but TestUpgrades skips that. + _, err := tx.CreateBucketIfNotExists(newActiveOrdersBucket) + if err != nil { + return err + } + + archivedOrdersBkt := tx.Bucket(oldOrdersBucket) + activeOrdersBkt := tx.Bucket(newActiveOrdersBucket) + return archivedOrdersBkt.ForEach(func(k, _ []byte) error { + archivedOBkt := archivedOrdersBkt.Bucket(k) + if archivedOBkt == nil { + return fmt.Errorf("order %x bucket is not a bucket", k) + } + status := order.OrderStatus(intCoder.Uint16(archivedOBkt.Get(statusKey))) + if status == order.OrderStatusUnknown { + fmt.Printf("Encountered order with unknown status: %x\n", k) + return nil + } + if !status.IsActive() { + return nil + } + activeOBkt, err := activeOrdersBkt.CreateBucket(k) + if err != nil { + return err + } + // Assume the order bucket contains only values, no sub-buckets + if err := archivedOBkt.ForEach(func(k, v []byte) error { + return activeOBkt.Put(k, v) + }); err != nil { + return err + } + return archivedOrdersBkt.DeleteBucket(k) + }) +} + func doUpgrade(tx *bbolt.Tx, upgrade upgradefunc, newVersion uint32) error { err := upgrade(tx) if err != nil { diff --git a/client/db/bolt/upgrades_test.go b/client/db/bolt/upgrades_test.go index 33fa608880..d12175926d 100644 --- a/client/db/bolt/upgrades_test.go +++ b/client/db/bolt/upgrades_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "decred.org/dcrdex/dex/order" + "go.etcd.io/bbolt" ) @@ -25,10 +27,11 @@ var dbUpgradeTests = [...]struct { filename string // in testdata directory newVersion uint32 }{ - // {"testnetbot", v1Upgrade, verifyV1Upgrade, "dexbot-testnet.db.gz", 3}, // only for TestUpgradeDB + // {"testnetbot", v4Upgrade, verifyV4Upgrade, "dexbot-testnet.db.gz", 4}, // only for TestUpgradeDB {"upgradeFromV0", v1Upgrade, verifyV1Upgrade, "v0.db.gz", 1}, {"upgradeFromV1", v2Upgrade, verifyV2Upgrade, "v1.db.gz", 2}, {"upgradeFromV2", v3Upgrade, verifyV3Upgrade, "v2.db.gz", 3}, + {"upgradeFromV3", v4Upgrade, verifyV4Upgrade, "v3.db.gz", 4}, } func TestUpgrades(t *testing.T) { @@ -128,6 +131,7 @@ func verifyV1Upgrade(t *testing.T, db *bbolt.DB) { func verifyV2Upgrade(t *testing.T, db *bbolt.DB) { t.Helper() maxFeeB := uint64Bytes(^uint64(0)) + ordersBucket := []byte("orders") err := db.View(func(dbtx *bbolt.Tx) error { err := checkVersion(dbtx, 2) @@ -167,6 +171,69 @@ func verifyV3Upgrade(t *testing.T, db *bbolt.DB) { } } +func verifyV4Upgrade(t *testing.T, db *bbolt.DB) { + oldOrdersBucket := []byte("orders") + newActiveOrdersBucket := []byte("activeOrders") + err := db.View(func(dbtx *bbolt.Tx) error { + err := checkVersion(dbtx, 4) + if err != nil { + return err + } + // Ensure we have both old and new buckets. + archivedOrdersBkt := dbtx.Bucket(oldOrdersBucket) + if archivedOrdersBkt == nil { + return fmt.Errorf("archived orders bucket not found") + } + activeOrdersBkt := dbtx.Bucket(newActiveOrdersBucket) + if activeOrdersBkt == nil { + return fmt.Errorf("active orders bucket not found") + } + + // Ensure the old bucket now only contains finished orders. + err = archivedOrdersBkt.ForEach(func(k, _ []byte) error { + archivedOBkt := archivedOrdersBkt.Bucket(k) + if archivedOBkt == nil { + return fmt.Errorf("order %x bucket is not a bucket", k) + } + status := order.OrderStatus(intCoder.Uint16(archivedOBkt.Get(statusKey))) + if status == order.OrderStatusUnknown { + fmt.Printf("Encountered order with unknown status: %x\n", k) + return nil + } + if status.IsActive() { + return fmt.Errorf("archived bucket has active order: %x", k) + } + return nil + }) + if err != nil { + return err + } + + // Ensure the new bucket only contains active orders. + err = activeOrdersBkt.ForEach(func(k, _ []byte) error { + activeOBkt := activeOrdersBkt.Bucket(k) + if activeOBkt == nil { + return fmt.Errorf("order %x bucket is not a bucket", k) + } + status := order.OrderStatus(intCoder.Uint16(activeOBkt.Get(statusKey))) + if status == order.OrderStatusUnknown { + return fmt.Errorf("encountered order with unknown status: %x", k) + } + if !status.IsActive() { + return fmt.Errorf("active orders bucket has archived order: %x", k) + } + return nil + }) + if err != nil { + return err + } + return nil + }) + if err != nil { + t.Error(err) + } +} + func checkVersion(dbtx *bbolt.Tx, expectedVersion uint32) error { bkt := dbtx.Bucket(appBucket) if bkt == nil { diff --git a/dex/order/status.go b/dex/order/status.go index e49fa6dfee..6e85f94f1a 100644 --- a/dex/order/status.go +++ b/dex/order/status.go @@ -74,3 +74,8 @@ func (s OrderStatus) String() string { } return name } + +// IsActive returns whether the status is considered active. +func (s OrderStatus) IsActive() bool { + return s == OrderStatusEpoch || s == OrderStatusBooked +}