From d20ab95acc853caef2eda7fe7e2ac26299e59776 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 25 Sep 2020 01:22:45 -0700 Subject: [PATCH 1/3] feat(piecestore): switch to cbor map encoding switch to map encoding for piecestore and setup migrations --- go.mod | 9 +- go.sum | 138 ++++- piecestore/impl/piecestore.go | 207 +++++++ piecestore/{ => impl}/piecestore_test.go | 184 +++++- piecestore/migrations/migrations.go | 89 +++ piecestore/migrations/migrations_cbor_gen.go | 471 +++++++++++++++ piecestore/piecestore.go | 153 ----- piecestore/types.go | 9 +- piecestore/types_cbor_gen.go | 570 ++++++++++++------ .../storage_retrieval_integration_test.go | 15 +- shared_testutil/test_piecestore.go | 8 + storagemarket/integration_test.go | 16 +- 12 files changed, 1518 insertions(+), 351 deletions(-) create mode 100644 piecestore/impl/piecestore.go rename piecestore/{ => impl}/piecestore_test.go (50%) create mode 100644 piecestore/migrations/migrations.go create mode 100644 piecestore/migrations/migrations_cbor_gen.go delete mode 100644 piecestore/piecestore.go diff --git a/go.mod b/go.mod index 92eb5a64..2cecebc2 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,11 @@ require ( github.com/filecoin-project/go-address v0.0.3 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-data-transfer v0.6.6 + github.com/filecoin-project/go-ds-versioning v0.0.0-20200925081648-2b6f19ff8a67 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df - github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 + github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b github.com/filecoin-project/specs-actors v0.9.7 @@ -21,7 +22,7 @@ require ( github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 github.com/ipfs/go-cid v0.0.7 - github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-datastore v0.4.5 github.com/ipfs/go-graphsync v0.2.1 github.com/ipfs/go-ipfs-blockstore v1.0.1 github.com/ipfs/go-ipfs-blocksutil v0.0.1 @@ -44,8 +45,8 @@ require ( github.com/multiformats/go-multibase v0.0.3 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e github.com/stretchr/testify v1.6.1 - github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee - golang.org/x/exp v0.0.0-20190121172915-509febef88a4 + github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163 + golang.org/x/exp v0.0.0-20200207192155-f17229e696bd golang.org/x/net v0.0.0-20200625001655-4c5254603344 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/go.sum b/go.sum index 480016ea..dbcebdf0 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,22 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= @@ -11,6 +26,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOv github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= @@ -39,9 +55,13 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -72,6 +92,8 @@ github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70d github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/filecoin-project/go-address v0.0.3 h1:eVfbdjEbpbzIrbiSa+PiGUY+oDK9HnUn+M1R/ggoHf8= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= @@ -84,6 +106,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.6.6 h1:2TccLSxPYJENcYRdov2WvpTvQ1qUMrPkWe8sBrfj36g= github.com/filecoin-project/go-data-transfer v0.6.6/go.mod h1:C++k1U6+jMQODOaen5OPDo9XQbth9Yq3ie94vNjBJbk= +github.com/filecoin-project/go-ds-versioning v0.0.0-20200925081648-2b6f19ff8a67 h1:+ahe+CxnXRJ2qp6VKzhHDs5+eBSQUeBadBrRyEjAITM= +github.com/filecoin-project/go-ds-versioning v0.0.0-20200925081648-2b6f19ff8a67/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI= @@ -96,8 +120,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df h1:m2esXSuGBkuXlRyCsl1a/7/FkFam63o1OzIgzaHtOfI= github.com/filecoin-project/go-state-types v0.0.0-20200905071437-95828685f9df/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= -github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c= -github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw= +github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= @@ -115,6 +139,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -123,15 +149,20 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -140,6 +171,7 @@ github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -151,11 +183,15 @@ github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbB github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= @@ -185,6 +221,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= @@ -221,6 +258,8 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg= +github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -346,6 +385,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -661,6 +701,7 @@ github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -671,6 +712,7 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -757,8 +799,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c h1:BMg3YUwLEUIYBJoYZVhA4ZDTciXRj6r7ffOCshWrsoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee h1:U7zWWvvAjT76EiuWPSOiZlQDnaQYPxPoxugTtTAcJK0= -github.com/whyrusleeping/cbor-gen v0.0.0-20200814224545-656e08ce49ee/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163 h1:TtcUeY2XZSriVWR1pXyfCBWIf/NGC2iUdNw1lofUjUU= +github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= @@ -781,6 +823,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= @@ -801,6 +844,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= +go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -814,6 +859,7 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -825,14 +871,34 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4 h1:c2HOrn5iMezYjSlGPncknSEr/8x5LELb/ilJbXi9DEA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -848,23 +914,33 @@ golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -877,20 +953,28 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -901,6 +985,7 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -910,15 +995,31 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 h1:r3P/5xOq/dK1991B65Oy6E1fRF/2d/fSYZJ/fXGVfJc= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -931,22 +1032,48 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -981,6 +1108,8 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ= @@ -993,6 +1122,7 @@ modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/piecestore/impl/piecestore.go b/piecestore/impl/piecestore.go new file mode 100644 index 00000000..76a7a266 --- /dev/null +++ b/piecestore/impl/piecestore.go @@ -0,0 +1,207 @@ +package piecestoreimpl + +import ( + "context" + + "github.com/hannahhoward/go-pubsub" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + versioning "github.com/filecoin-project/go-ds-versioning/pkg" + versioned "github.com/filecoin-project/go-ds-versioning/pkg/statestore" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-fil-markets/piecestore/migrations" +) + +var log = logging.Logger("piecestore") + +// DSPiecePrefix is the name space for storing piece infos +var DSPiecePrefix = "/pieces" + +// DSCIDPrefix is the name space for storing CID infos +var DSCIDPrefix = "/cid-infos" + +// NewPieceStore returns a new piecestore based on the given datastore +func NewPieceStore(ds datastore.Batching) (piecestore.PieceStore, error) { + pieceInfoMigrations, err := migrations.PieceInfoMigrations.Build() + if err != nil { + return nil, err + } + pieces, migratePieces := versioned.NewVersionedStateStore(namespace.Wrap(ds, datastore.NewKey(DSPiecePrefix)), pieceInfoMigrations, versioning.VersionKey("1")) + cidInfoMigrations, err := migrations.CIDInfoMigrations.Build() + if err != nil { + return nil, err + } + cidInfos, migrateCidInfos := versioned.NewVersionedStateStore(namespace.Wrap(ds, datastore.NewKey(DSCIDPrefix)), cidInfoMigrations, versioning.VersionKey("1")) + return &pieceStore{ + readySub: pubsub.New(readyDispatcher), + pieces: pieces, + migratePieces: migratePieces, + cidInfos: cidInfos, + migrateCidInfos: migrateCidInfos, + }, nil +} + +type pieceStore struct { + readySub *pubsub.PubSub + migratePieces func(ctx context.Context) error + pieces versioned.StateStore + migrateCidInfos func(ctx context.Context) error + cidInfos versioned.StateStore +} + +func readyDispatcher(_ pubsub.Event, fn pubsub.SubscriberFn) error { + cb, ok := fn.(piecestore.ReadyFunc) + if !ok { + return xerrors.New("wrong type of event") + } + cb() + return nil +} + +func (ps *pieceStore) Start(ctx context.Context) error { + go func() { + err := ps.migratePieces(ctx) + if err != nil { + log.Errorf("Migrating pieceInfos: %s", err.Error()) + } + err = ps.migrateCidInfos(ctx) + if err != nil { + log.Errorf("Migrating cidInfos: %s", err.Error()) + } + ps.readySub.Publish(struct{}{}) + }() + return nil +} + +func (ps *pieceStore) OnReady(ready piecestore.ReadyFunc) { + ps.readySub.Subscribe(ready) +} + +// Store `dealInfo` in the PieceStore with key `pieceCID`. +func (ps *pieceStore) AddDealForPiece(pieceCID cid.Cid, dealInfo piecestore.DealInfo) error { + return ps.mutatePieceInfo(pieceCID, func(pi *piecestore.PieceInfo) error { + for _, di := range pi.Deals { + if di == dealInfo { + return nil + } + } + pi.Deals = append(pi.Deals, dealInfo) + return nil + }) +} + +// Store the map of blockLocations in the PieceStore's CIDInfo store, with key `pieceCID` +func (ps *pieceStore) AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]piecestore.BlockLocation) error { + for c, blockLocation := range blockLocations { + err := ps.mutateCIDInfo(c, func(ci *piecestore.CIDInfo) error { + for _, pbl := range ci.PieceBlockLocations { + if pbl.PieceCID.Equals(pieceCID) && pbl.BlockLocation == blockLocation { + return nil + } + } + ci.PieceBlockLocations = append(ci.PieceBlockLocations, piecestore.PieceBlockLocation{BlockLocation: blockLocation, PieceCID: pieceCID}) + return nil + }) + if err != nil { + return err + } + } + return nil +} + +func (ps *pieceStore) ListPieceInfoKeys() ([]cid.Cid, error) { + var pis []piecestore.PieceInfo + if err := ps.pieces.List(&pis); err != nil { + return nil, err + } + + out := make([]cid.Cid, 0, len(pis)) + for _, pi := range pis { + out = append(out, pi.PieceCID) + } + + return out, nil +} + +func (ps *pieceStore) ListCidInfoKeys() ([]cid.Cid, error) { + var cis []piecestore.CIDInfo + if err := ps.cidInfos.List(&cis); err != nil { + return nil, err + } + + out := make([]cid.Cid, 0, len(cis)) + for _, ci := range cis { + out = append(out, ci.CID) + } + + return out, nil +} + +// Retrieve the PieceInfo associated with `pieceCID` from the piece info store. +func (ps *pieceStore) GetPieceInfo(pieceCID cid.Cid) (piecestore.PieceInfo, error) { + var out piecestore.PieceInfo + if err := ps.pieces.Get(pieceCID).Get(&out); err != nil { + return piecestore.PieceInfo{}, err + } + return out, nil +} + +// Retrieve the CIDInfo associated with `pieceCID` from the CID info store. +func (ps *pieceStore) GetCIDInfo(payloadCID cid.Cid) (piecestore.CIDInfo, error) { + var out piecestore.CIDInfo + if err := ps.cidInfos.Get(payloadCID).Get(&out); err != nil { + return piecestore.CIDInfo{}, err + } + return out, nil +} + +func (ps *pieceStore) ensurePieceInfo(pieceCID cid.Cid) error { + has, err := ps.pieces.Has(pieceCID) + + if err != nil { + return err + } + if has { + return nil + } + + pieceInfo := piecestore.PieceInfo{PieceCID: pieceCID} + return ps.pieces.Begin(pieceCID, &pieceInfo) +} + +func (ps *pieceStore) ensureCIDInfo(c cid.Cid) error { + has, err := ps.cidInfos.Has(c) + + if err != nil { + return err + } + + if has { + return nil + } + + cidInfo := piecestore.CIDInfo{CID: c} + return ps.cidInfos.Begin(c, &cidInfo) +} + +func (ps *pieceStore) mutatePieceInfo(pieceCID cid.Cid, mutator interface{}) error { + err := ps.ensurePieceInfo(pieceCID) + if err != nil { + return err + } + + return ps.pieces.Get(pieceCID).Mutate(mutator) +} + +func (ps *pieceStore) mutateCIDInfo(c cid.Cid, mutator interface{}) error { + err := ps.ensureCIDInfo(c) + if err != nil { + return err + } + + return ps.cidInfos.Get(c).Mutate(mutator) +} diff --git a/piecestore/piecestore_test.go b/piecestore/impl/piecestore_test.go similarity index 50% rename from piecestore/piecestore_test.go rename to piecestore/impl/piecestore_test.go index 52de5e0c..2512405f 100644 --- a/piecestore/piecestore_test.go +++ b/piecestore/impl/piecestore_test.go @@ -1,32 +1,53 @@ -package piecestore_test +package piecestoreimpl_test import ( + "context" "math/rand" "testing" + "time" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-fil-markets/piecestore" + piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl" + "github.com/filecoin-project/go-fil-markets/piecestore/migrations" "github.com/filecoin-project/go-fil-markets/shared_testutil" ) func TestStorePieceInfo(t *testing.T) { - + ctx := context.Background() pieceCid := shared_testutil.GenerateCids(1)[0] - initializePieceStore := func(t *testing.T) piecestore.PieceStore { - ps := piecestore.NewPieceStore(datastore.NewMapDatastore()) - _, err := ps.GetPieceInfo(pieceCid) + initializePieceStore := func(t *testing.T, ctx context.Context) piecestore.PieceStore { + ps, err := piecestoreimpl.NewPieceStore(datastore.NewMapDatastore()) + require.NoError(t, err) + ready := make(chan struct{}) + ps.OnReady(func() { + close(ready) + }) + err = ps.Start(ctx) + assert.NoError(t, err) + select { + case <-ctx.Done(): + t.Error("did not finish migrating") + case <-ready: + } + _, err = ps.GetPieceInfo(pieceCid) assert.Error(t, err) return ps } // Add a deal info t.Run("can add deals", func(t *testing.T) { - ps := initializePieceStore(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ps := initializePieceStore(t, ctx) dealInfo := piecestore.DealInfo{ DealID: abi.DealID(rand.Uint64()), SectorID: abi.SectorNumber(rand.Uint64()), @@ -43,7 +64,9 @@ func TestStorePieceInfo(t *testing.T) { }) t.Run("adding same deal twice does not dup", func(t *testing.T) { - ps := initializePieceStore(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ps := initializePieceStore(t, ctx) dealInfo := piecestore.DealInfo{ DealID: abi.DealID(rand.Uint64()), SectorID: abi.SectorNumber(rand.Uint64()), @@ -69,6 +92,7 @@ func TestStorePieceInfo(t *testing.T) { } func TestStoreCIDInfo(t *testing.T) { + ctx := context.Background() pieceCids := shared_testutil.GenerateCids(2) pieceCid1 := pieceCids[0] pieceCid2 := pieceCids[1] @@ -81,15 +105,29 @@ func TestStoreCIDInfo(t *testing.T) { }) } - initializePieceStore := func(t *testing.T) piecestore.PieceStore { - ps := piecestore.NewPieceStore(datastore.NewMapDatastore()) - _, err := ps.GetCIDInfo(testCIDs[0]) + initializePieceStore := func(t *testing.T, ctx context.Context) piecestore.PieceStore { + ps, err := piecestoreimpl.NewPieceStore(datastore.NewMapDatastore()) + assert.NoError(t, err) + ready := make(chan struct{}) + ps.OnReady(func() { + close(ready) + }) + err = ps.Start(ctx) + assert.NoError(t, err) + select { + case <-ctx.Done(): + t.Error("did not finish migrating") + case <-ready: + } + _, err = ps.GetCIDInfo(testCIDs[0]) assert.Error(t, err) return ps } t.Run("can add piece block locations", func(t *testing.T) { - ps := initializePieceStore(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ps := initializePieceStore(t, ctx) err := ps.AddPieceBlockLocations(pieceCid1, map[cid.Cid]piecestore.BlockLocation{ testCIDs[0]: blockLocations[0], testCIDs[1]: blockLocations[1], @@ -114,7 +152,9 @@ func TestStoreCIDInfo(t *testing.T) { }) t.Run("overlapping adds", func(t *testing.T) { - ps := initializePieceStore(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ps := initializePieceStore(t, ctx) err := ps.AddPieceBlockLocations(pieceCid1, map[cid.Cid]piecestore.BlockLocation{ testCIDs[0]: blockLocations[0], testCIDs[1]: blockLocations[2], @@ -144,7 +184,9 @@ func TestStoreCIDInfo(t *testing.T) { }) t.Run("duplicate adds", func(t *testing.T) { - ps := initializePieceStore(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ps := initializePieceStore(t, ctx) err := ps.AddPieceBlockLocations(pieceCid1, map[cid.Cid]piecestore.BlockLocation{ testCIDs[0]: blockLocations[0], testCIDs[1]: blockLocations[1], @@ -172,3 +214,119 @@ func TestStoreCIDInfo(t *testing.T) { assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid1}) }) } + +func TestMigrations(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + pieceCids := shared_testutil.GenerateCids(1) + pieceCid1 := pieceCids[0] + testCIDs := shared_testutil.GenerateCids(3) + blockLocations := make([]piecestore.BlockLocation, 0, 3) + for i := 0; i < 3; i++ { + blockLocations = append(blockLocations, piecestore.BlockLocation{ + RelOffset: rand.Uint64(), + BlockSize: rand.Uint64(), + }) + } + dealInfo := piecestore.DealInfo{ + DealID: abi.DealID(rand.Uint64()), + SectorID: abi.SectorNumber(rand.Uint64()), + Offset: abi.PaddedPieceSize(rand.Uint64()), + Length: abi.PaddedPieceSize(rand.Uint64()), + } + + ds := datastore.NewMapDatastore() + + oldCidInfos := statestore.New(namespace.Wrap(ds, datastore.NewKey(piecestoreimpl.DSCIDPrefix))) + err := oldCidInfos.Begin(testCIDs[0], &migrations.CIDInfo0{ + CID: testCIDs[0], + PieceBlockLocations: []migrations.PieceBlockLocation0{ + { + BlockLocation0: migrations.BlockLocation0{ + RelOffset: blockLocations[0].RelOffset, + BlockSize: blockLocations[0].BlockSize, + }, + PieceCID: pieceCid1, + }, + }, + }) + require.NoError(t, err) + err = oldCidInfos.Begin(testCIDs[1], &migrations.CIDInfo0{ + CID: testCIDs[1], + PieceBlockLocations: []migrations.PieceBlockLocation0{ + { + BlockLocation0: migrations.BlockLocation0{ + RelOffset: blockLocations[1].RelOffset, + BlockSize: blockLocations[1].BlockSize, + }, + PieceCID: pieceCid1, + }, + }, + }) + require.NoError(t, err) + err = oldCidInfos.Begin(testCIDs[2], &migrations.CIDInfo0{ + CID: testCIDs[2], + PieceBlockLocations: []migrations.PieceBlockLocation0{ + { + BlockLocation0: migrations.BlockLocation0{ + RelOffset: blockLocations[2].RelOffset, + BlockSize: blockLocations[2].BlockSize, + }, + PieceCID: pieceCid1, + }, + }, + }) + require.NoError(t, err) + oldPieces := statestore.New(namespace.Wrap(ds, datastore.NewKey(piecestoreimpl.DSPiecePrefix))) + err = oldPieces.Begin(pieceCid1, &migrations.PieceInfo0{ + PieceCID: pieceCid1, + Deals: []migrations.DealInfo0{ + { + DealID: dealInfo.DealID, + SectorID: dealInfo.SectorID, + Offset: dealInfo.Offset, + Length: dealInfo.Length, + }, + }, + }) + require.NoError(t, err) + + ps, err := piecestoreimpl.NewPieceStore(ds) + require.NoError(t, err) + ready := make(chan struct{}) + ps.OnReady(func() { + close(ready) + }) + err = ps.Start(ctx) + assert.NoError(t, err) + select { + case <-ctx.Done(): + t.Error("did not finish migrating") + case <-ready: + } + + t.Run("migrates deals", func(t *testing.T) { + pi, err := ps.GetPieceInfo(pieceCid1) + assert.NoError(t, err) + assert.Len(t, pi.Deals, 1) + assert.Equal(t, pi.Deals[0], dealInfo) + }) + + t.Run("migrates piece block locations", func(t *testing.T) { + ci, err := ps.GetCIDInfo(testCIDs[0]) + assert.NoError(t, err) + assert.Len(t, ci.PieceBlockLocations, 1) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[0], pieceCid1}) + + ci, err = ps.GetCIDInfo(testCIDs[1]) + assert.NoError(t, err) + assert.Len(t, ci.PieceBlockLocations, 1) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[1], pieceCid1}) + + ci, err = ps.GetCIDInfo(testCIDs[2]) + assert.NoError(t, err) + assert.Len(t, ci.PieceBlockLocations, 1) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid1}) + }) +} diff --git a/piecestore/migrations/migrations.go b/piecestore/migrations/migrations.go new file mode 100644 index 00000000..7399f7eb --- /dev/null +++ b/piecestore/migrations/migrations.go @@ -0,0 +1,89 @@ +package migrations + +import ( + versioning "github.com/filecoin-project/go-ds-versioning/pkg" + "github.com/filecoin-project/go-ds-versioning/pkg/versioned" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" +) + +//go:generate cbor-gen-for PieceInfo0 DealInfo0 BlockLocation0 PieceBlockLocation0 CIDInfo0 + +// DealInfo0 is version 0 of DealInfo +type DealInfo0 struct { + DealID abi.DealID + SectorID abi.SectorNumber + Offset abi.PaddedPieceSize + Length abi.PaddedPieceSize +} + +// BlockLocation0 is version 0 of BlockLocation +type BlockLocation0 struct { + RelOffset uint64 + BlockSize uint64 +} + +// PieceBlockLocation0 is version 0 of PieceBlockLocation +// is inside of +type PieceBlockLocation0 struct { + BlockLocation0 + PieceCID cid.Cid +} + +// CIDInfo0 is version 0 of CIDInfo +type CIDInfo0 struct { + CID cid.Cid + PieceBlockLocations []PieceBlockLocation0 +} + +// PieceInfo0 is version 0 of PieceInfo +type PieceInfo0 struct { + PieceCID cid.Cid + Deals []DealInfo0 +} + +// MigratePieceInfo0to1 migrates a tuple encoded piece info to a map encoded piece info +func MigratePieceInfo0to1(oldPi *PieceInfo0) (*piecestore.PieceInfo, error) { + deals := make([]piecestore.DealInfo, len(oldPi.Deals)) + for i, oldDi := range oldPi.Deals { + deals[i] = piecestore.DealInfo{ + DealID: oldDi.DealID, + SectorID: oldDi.SectorID, + Offset: oldDi.Offset, + Length: oldDi.Length, + } + } + return &piecestore.PieceInfo{ + PieceCID: oldPi.PieceCID, + Deals: deals, + }, nil +} + +// MigrateCidInfo0to1 migrates a tuple encoded cid info to a map encoded cid info +func MigrateCidInfo0to1(oldCi *CIDInfo0) (*piecestore.CIDInfo, error) { + pieceBlockLocations := make([]piecestore.PieceBlockLocation, len(oldCi.PieceBlockLocations)) + for i, oldPbl := range oldCi.PieceBlockLocations { + pieceBlockLocations[i] = piecestore.PieceBlockLocation{ + BlockLocation: piecestore.BlockLocation{ + RelOffset: oldPbl.RelOffset, + BlockSize: oldPbl.BlockSize, + }, + PieceCID: oldPbl.PieceCID, + } + } + return &piecestore.CIDInfo{ + CID: oldCi.CID, + PieceBlockLocations: pieceBlockLocations, + }, nil +} + +// PieceInfoMigrations is the list of migrations for migrating PieceInfos +var PieceInfoMigrations = versioned.BuilderList{ + versioned.NewVersionedBuilder(MigratePieceInfo0to1, versioning.VersionKey("1")), +} + +// CIDInfoMigrations is the list of migrations for migrating CIDInfos +var CIDInfoMigrations = versioned.BuilderList{ + versioned.NewVersionedBuilder(MigrateCidInfo0to1, versioning.VersionKey("1")), +} diff --git a/piecestore/migrations/migrations_cbor_gen.go b/piecestore/migrations/migrations_cbor_gen.go new file mode 100644 index 00000000..e981126c --- /dev/null +++ b/piecestore/migrations/migrations_cbor_gen.go @@ -0,0 +1,471 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package migrations + +import ( + "fmt" + "io" + + abi "github.com/filecoin-project/go-state-types/abi" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf + +var lengthBufPieceInfo0 = []byte{130} + +func (t *PieceInfo0) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufPieceInfo0); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.PieceCID (cid.Cid) (struct) + + if err := cbg.WriteCidBuf(scratch, w, t.PieceCID); err != nil { + return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) + } + + // t.Deals ([]migrations.DealInfo0) (slice) + if len(t.Deals) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Deals was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Deals))); err != nil { + return err + } + for _, v := range t.Deals { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *PieceInfo0) UnmarshalCBOR(r io.Reader) error { + *t = PieceInfo0{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.PieceCID (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } + + t.PieceCID = c + + } + // t.Deals ([]migrations.DealInfo0) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Deals: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Deals = make([]DealInfo0, extra) + } + + for i := 0; i < int(extra); i++ { + + var v DealInfo0 + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Deals[i] = v + } + + return nil +} + +var lengthBufDealInfo0 = []byte{132} + +func (t *DealInfo0) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufDealInfo0); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.DealID (abi.DealID) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.SectorID (abi.SectorNumber) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil { + return err + } + + // t.Offset (abi.PaddedPieceSize) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Offset)); err != nil { + return err + } + + // t.Length (abi.PaddedPieceSize) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Length)); err != nil { + return err + } + + return nil +} + +func (t *DealInfo0) UnmarshalCBOR(r io.Reader) error { + *t = DealInfo0{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.DealID (abi.DealID) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = abi.DealID(extra) + + } + // t.SectorID (abi.SectorNumber) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = abi.SectorNumber(extra) + + } + // t.Offset (abi.PaddedPieceSize) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = abi.PaddedPieceSize(extra) + + } + // t.Length (abi.PaddedPieceSize) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Length = abi.PaddedPieceSize(extra) + + } + return nil +} + +var lengthBufBlockLocation0 = []byte{130} + +func (t *BlockLocation0) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufBlockLocation0); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.RelOffset (uint64) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.RelOffset)); err != nil { + return err + } + + // t.BlockSize (uint64) (uint64) + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.BlockSize)); err != nil { + return err + } + + return nil +} + +func (t *BlockLocation0) UnmarshalCBOR(r io.Reader) error { + *t = BlockLocation0{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.RelOffset (uint64) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.RelOffset = uint64(extra) + + } + // t.BlockSize (uint64) (uint64) + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockSize = uint64(extra) + + } + return nil +} + +var lengthBufPieceBlockLocation0 = []byte{130} + +func (t *PieceBlockLocation0) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufPieceBlockLocation0); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.BlockLocation0 (migrations.BlockLocation0) (struct) + if err := t.BlockLocation0.MarshalCBOR(w); err != nil { + return err + } + + // t.PieceCID (cid.Cid) (struct) + + if err := cbg.WriteCidBuf(scratch, w, t.PieceCID); err != nil { + return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) + } + + return nil +} + +func (t *PieceBlockLocation0) UnmarshalCBOR(r io.Reader) error { + *t = PieceBlockLocation0{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.BlockLocation0 (migrations.BlockLocation0) (struct) + + { + + if err := t.BlockLocation0.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.BlockLocation0: %w", err) + } + + } + // t.PieceCID (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } + + t.PieceCID = c + + } + return nil +} + +var lengthBufCIDInfo0 = []byte{130} + +func (t *CIDInfo0) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufCIDInfo0); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.CID (cid.Cid) (struct) + + if err := cbg.WriteCidBuf(scratch, w, t.CID); err != nil { + return xerrors.Errorf("failed to write cid field t.CID: %w", err) + } + + // t.PieceBlockLocations ([]migrations.PieceBlockLocation0) (slice) + if len(t.PieceBlockLocations) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.PieceBlockLocations was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.PieceBlockLocations))); err != nil { + return err + } + for _, v := range t.PieceBlockLocations { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *CIDInfo0) UnmarshalCBOR(r io.Reader) error { + *t = CIDInfo0{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.CID (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CID: %w", err) + } + + t.CID = c + + } + // t.PieceBlockLocations ([]migrations.PieceBlockLocation0) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.PieceBlockLocations: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.PieceBlockLocations = make([]PieceBlockLocation0, extra) + } + + for i := 0; i < int(extra); i++ { + + var v PieceBlockLocation0 + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.PieceBlockLocations[i] = v + } + + return nil +} diff --git a/piecestore/piecestore.go b/piecestore/piecestore.go deleted file mode 100644 index adefcfc4..00000000 --- a/piecestore/piecestore.go +++ /dev/null @@ -1,153 +0,0 @@ -package piecestore - -import ( - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - - "github.com/filecoin-project/go-statestore" -) - -// DSPiecePrefix is the name space for storing piece infos -var DSPiecePrefix = "/pieces" - -// DSCIDPrefix is the name space for storing CID infos -var DSCIDPrefix = "/cid-infos" - -// NewPieceStore returns a new piecestore based on the given datastore -func NewPieceStore(ds datastore.Batching) PieceStore { - return &pieceStore{ - pieces: statestore.New(namespace.Wrap(ds, datastore.NewKey(DSPiecePrefix))), - cidInfos: statestore.New(namespace.Wrap(ds, datastore.NewKey(DSCIDPrefix))), - } -} - -type pieceStore struct { - pieces *statestore.StateStore - cidInfos *statestore.StateStore -} - -// Store `dealInfo` in the PieceStore with key `pieceCID`. -func (ps *pieceStore) AddDealForPiece(pieceCID cid.Cid, dealInfo DealInfo) error { - return ps.mutatePieceInfo(pieceCID, func(pi *PieceInfo) error { - for _, di := range pi.Deals { - if di == dealInfo { - return nil - } - } - pi.Deals = append(pi.Deals, dealInfo) - return nil - }) -} - -// Store the map of blockLocations in the PieceStore's CIDInfo store, with key `pieceCID` -func (ps *pieceStore) AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]BlockLocation) error { - for c, blockLocation := range blockLocations { - err := ps.mutateCIDInfo(c, func(ci *CIDInfo) error { - for _, pbl := range ci.PieceBlockLocations { - if pbl.PieceCID.Equals(pieceCID) && pbl.BlockLocation == blockLocation { - return nil - } - } - ci.PieceBlockLocations = append(ci.PieceBlockLocations, PieceBlockLocation{blockLocation, pieceCID}) - return nil - }) - if err != nil { - return err - } - } - return nil -} - -func (ps *pieceStore) ListPieceInfoKeys() ([]cid.Cid, error) { - var pis []PieceInfo - if err := ps.pieces.List(&pis); err != nil { - return nil, err - } - - out := make([]cid.Cid, 0, len(pis)) - for _, pi := range pis { - out = append(out, pi.PieceCID) - } - - return out, nil -} - -func (ps *pieceStore) ListCidInfoKeys() ([]cid.Cid, error) { - var cis []CIDInfo - if err := ps.cidInfos.List(&cis); err != nil { - return nil, err - } - - out := make([]cid.Cid, 0, len(cis)) - for _, ci := range cis { - out = append(out, ci.CID) - } - - return out, nil -} - -// Retrieve the PieceInfo associated with `pieceCID` from the piece info store. -func (ps *pieceStore) GetPieceInfo(pieceCID cid.Cid) (PieceInfo, error) { - var out PieceInfo - if err := ps.pieces.Get(pieceCID).Get(&out); err != nil { - return PieceInfo{}, err - } - return out, nil -} - -// Retrieve the CIDInfo associated with `pieceCID` from the CID info store. -func (ps *pieceStore) GetCIDInfo(payloadCID cid.Cid) (CIDInfo, error) { - var out CIDInfo - if err := ps.cidInfos.Get(payloadCID).Get(&out); err != nil { - return CIDInfo{}, err - } - return out, nil -} - -func (ps *pieceStore) ensurePieceInfo(pieceCID cid.Cid) error { - has, err := ps.pieces.Has(pieceCID) - - if err != nil { - return err - } - if has { - return nil - } - - pieceInfo := PieceInfo{PieceCID: pieceCID} - return ps.pieces.Begin(pieceCID, &pieceInfo) -} - -func (ps *pieceStore) ensureCIDInfo(c cid.Cid) error { - has, err := ps.cidInfos.Has(c) - - if err != nil { - return err - } - - if has { - return nil - } - - cidInfo := CIDInfo{CID: c} - return ps.cidInfos.Begin(c, &cidInfo) -} - -func (ps *pieceStore) mutatePieceInfo(pieceCID cid.Cid, mutator interface{}) error { - err := ps.ensurePieceInfo(pieceCID) - if err != nil { - return err - } - - return ps.pieces.Get(pieceCID).Mutate(mutator) -} - -func (ps *pieceStore) mutateCIDInfo(c cid.Cid, mutator interface{}) error { - err := ps.ensureCIDInfo(c) - if err != nil { - return err - } - - return ps.cidInfos.Get(c).Mutate(mutator) -} diff --git a/piecestore/types.go b/piecestore/types.go index 4a4fae9c..2b1af81a 100644 --- a/piecestore/types.go +++ b/piecestore/types.go @@ -1,12 +1,14 @@ package piecestore import ( + "context" + "github.com/ipfs/go-cid" "github.com/filecoin-project/go-state-types/abi" ) -//go:generate cbor-gen-for PieceInfo DealInfo BlockLocation PieceBlockLocation CIDInfo +//go:generate cbor-gen-for --map-encoding PieceInfo DealInfo BlockLocation PieceBlockLocation CIDInfo // DealInfo is information about a single deal for a given piece type DealInfo struct { @@ -49,8 +51,13 @@ type PieceInfo struct { // PieceInfoUndefined is piece info with no information var PieceInfoUndefined = PieceInfo{} +// ReadyFunc is a callback when the piecestore is ready +type ReadyFunc func() + // PieceStore is a saved database of piece info that can be modified and queried type PieceStore interface { + Start(ctx context.Context) error + OnReady(ready ReadyFunc) AddDealForPiece(pieceCID cid.Cid, dealInfo DealInfo) error AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]BlockLocation) error GetPieceInfo(pieceCID cid.Cid) (PieceInfo, error) diff --git a/piecestore/types_cbor_gen.go b/piecestore/types_cbor_gen.go index 3596d96d..f01eaf02 100644 --- a/piecestore/types_cbor_gen.go +++ b/piecestore/types_cbor_gen.go @@ -13,26 +13,45 @@ import ( var _ = xerrors.Errorf -var lengthBufPieceInfo = []byte{130} - func (t *PieceInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufPieceInfo); err != nil { + if _, err := w.Write([]byte{162}); err != nil { return err } scratch := make([]byte, 9) // t.PieceCID (cid.Cid) (struct) + if len("PieceCID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PieceCID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PieceCID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("PieceCID")); err != nil { + return err + } if err := cbg.WriteCidBuf(scratch, w, t.PieceCID); err != nil { return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) } // t.Deals ([]piecestore.DealInfo) (slice) + if len("Deals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Deals\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Deals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Deals")); err != nil { + return err + } + if len(t.Deals) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.Deals was too long") } @@ -58,90 +77,149 @@ func (t *PieceInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("PieceInfo: map struct too large (%d)", extra) } - // t.PieceCID (cid.Cid) (struct) + var name string + n := extra + + for i := uint64(0); i < n; i++ { - { + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + name = string(sval) } - t.PieceCID = c + switch name { + // t.PieceCID (cid.Cid) (struct) + case "PieceCID": - } - // t.Deals ([]piecestore.DealInfo) (slice) + { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } - if extra > cbg.MaxLength { - return fmt.Errorf("t.Deals: array too large (%d)", extra) - } + t.PieceCID = c - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } + } + // t.Deals ([]piecestore.DealInfo) (slice) + case "Deals": - if extra > 0 { - t.Deals = make([]DealInfo, extra) - } + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } - for i := 0; i < int(extra); i++ { + if extra > cbg.MaxLength { + return fmt.Errorf("t.Deals: array too large (%d)", extra) + } - var v DealInfo - if err := v.UnmarshalCBOR(br); err != nil { - return err - } + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } - t.Deals[i] = v + if extra > 0 { + t.Deals = make([]DealInfo, extra) + } + + for i := 0; i < int(extra); i++ { + + var v DealInfo + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Deals[i] = v + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } } return nil } - -var lengthBufDealInfo = []byte{132} - func (t *DealInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufDealInfo); err != nil { + if _, err := w.Write([]byte{164}); err != nil { return err } scratch := make([]byte, 9) // t.DealID (abi.DealID) (uint64) + if len("DealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealID")); err != nil { + return err + } if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { return err } // t.SectorID (abi.SectorNumber) (uint64) + if len("SectorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("SectorID")); err != nil { + return err + } if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil { return err } // t.Offset (abi.PaddedPieceSize) (uint64) + if len("Offset") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Offset\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Offset")); err != nil { + return err + } if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Offset)); err != nil { return err } // t.Length (abi.PaddedPieceSize) (uint64) + if len("Length") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Length\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Length"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Length")); err != nil { + return err + } if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Length)); err != nil { return err @@ -160,93 +238,135 @@ func (t *DealInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") } - if extra != 4 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("DealInfo: map struct too large (%d)", extra) } - // t.DealID (abi.DealID) (uint64) - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = abi.DealID(extra) + var name string + n := extra - } - // t.SectorID (abi.SectorNumber) (uint64) + for i := uint64(0); i < n; i++ { - { + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err + name = string(sval) } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = abi.SectorNumber(extra) - } - // t.Offset (abi.PaddedPieceSize) (uint64) - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err + switch name { + // t.DealID (abi.DealID) (uint64) + case "DealID": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = abi.DealID(extra) + + } + // t.SectorID (abi.SectorNumber) (uint64) + case "SectorID": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = abi.SectorNumber(extra) + + } + // t.Offset (abi.PaddedPieceSize) (uint64) + case "Offset": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = abi.PaddedPieceSize(extra) + + } + // t.Length (abi.PaddedPieceSize) (uint64) + case "Length": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Length = abi.PaddedPieceSize(extra) + + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Offset = abi.PaddedPieceSize(extra) - } - // t.Length (abi.PaddedPieceSize) (uint64) - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Length = abi.PaddedPieceSize(extra) - } return nil } - -var lengthBufBlockLocation = []byte{130} - func (t *BlockLocation) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufBlockLocation); err != nil { + if _, err := w.Write([]byte{162}); err != nil { return err } scratch := make([]byte, 9) // t.RelOffset (uint64) (uint64) + if len("RelOffset") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RelOffset\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("RelOffset"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RelOffset")); err != nil { + return err + } if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.RelOffset)); err != nil { return err } // t.BlockSize (uint64) (uint64) + if len("BlockSize") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BlockSize\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("BlockSize"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("BlockSize")); err != nil { + return err + } if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.BlockSize)); err != nil { return err @@ -265,64 +385,105 @@ func (t *BlockLocation) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("BlockLocation: map struct too large (%d)", extra) } - // t.RelOffset (uint64) (uint64) + var name string + n := extra - { + for i := uint64(0); i < n; i++ { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + name = string(sval) } - t.RelOffset = uint64(extra) - } - // t.BlockSize (uint64) (uint64) + switch name { + // t.RelOffset (uint64) (uint64) + case "RelOffset": - { + { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.BlockSize = uint64(extra) + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.RelOffset = uint64(extra) + + } + // t.BlockSize (uint64) (uint64) + case "BlockSize": + + { + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockSize = uint64(extra) + + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } } + return nil } - -var lengthBufPieceBlockLocation = []byte{130} - func (t *PieceBlockLocation) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufPieceBlockLocation); err != nil { + if _, err := w.Write([]byte{162}); err != nil { return err } scratch := make([]byte, 9) // t.BlockLocation (piecestore.BlockLocation) (struct) + if len("BlockLocation") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BlockLocation\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("BlockLocation"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("BlockLocation")); err != nil { + return err + } + if err := t.BlockLocation.MarshalCBOR(w); err != nil { return err } // t.PieceCID (cid.Cid) (struct) + if len("PieceCID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PieceCID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PieceCID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("PieceCID")); err != nil { + return err + } if err := cbg.WriteCidBuf(scratch, w, t.PieceCID); err != nil { return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) @@ -341,58 +502,99 @@ func (t *PieceBlockLocation) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("PieceBlockLocation: map struct too large (%d)", extra) } - // t.BlockLocation (piecestore.BlockLocation) (struct) + var name string + n := extra + + for i := uint64(0); i < n; i++ { - { + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } - if err := t.BlockLocation.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.BlockLocation: %w", err) + name = string(sval) } - } - // t.PieceCID (cid.Cid) (struct) + switch name { + // t.BlockLocation (piecestore.BlockLocation) (struct) + case "BlockLocation": - { + { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) - } + if err := t.BlockLocation.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.BlockLocation: %w", err) + } + + } + // t.PieceCID (cid.Cid) (struct) + case "PieceCID": + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } - t.PieceCID = c + t.PieceCID = c + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } } + return nil } - -var lengthBufCIDInfo = []byte{130} - func (t *CIDInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufCIDInfo); err != nil { + if _, err := w.Write([]byte{162}); err != nil { return err } scratch := make([]byte, 9) // t.CID (cid.Cid) (struct) + if len("CID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("CID")); err != nil { + return err + } if err := cbg.WriteCidBuf(scratch, w, t.CID); err != nil { return xerrors.Errorf("failed to write cid field t.CID: %w", err) } // t.PieceBlockLocations ([]piecestore.PieceBlockLocation) (slice) + if len("PieceBlockLocations") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PieceBlockLocations\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PieceBlockLocations"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("PieceBlockLocations")); err != nil { + return err + } + if len(t.PieceBlockLocations) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.PieceBlockLocations was too long") } @@ -418,53 +620,75 @@ func (t *CIDInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("CIDInfo: map struct too large (%d)", extra) } - // t.CID (cid.Cid) (struct) + var name string + n := extra + + for i := uint64(0); i < n; i++ { - { + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.CID: %w", err) + name = string(sval) } - t.CID = c + switch name { + // t.CID (cid.Cid) (struct) + case "CID": - } - // t.PieceBlockLocations ([]piecestore.PieceBlockLocation) (slice) + { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CID: %w", err) + } - if extra > cbg.MaxLength { - return fmt.Errorf("t.PieceBlockLocations: array too large (%d)", extra) - } + t.CID = c - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } + } + // t.PieceBlockLocations ([]piecestore.PieceBlockLocation) (slice) + case "PieceBlockLocations": - if extra > 0 { - t.PieceBlockLocations = make([]PieceBlockLocation, extra) - } + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } - for i := 0; i < int(extra); i++ { + if extra > cbg.MaxLength { + return fmt.Errorf("t.PieceBlockLocations: array too large (%d)", extra) + } - var v PieceBlockLocation - if err := v.UnmarshalCBOR(br); err != nil { - return err - } + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } - t.PieceBlockLocations[i] = v + if extra > 0 { + t.PieceBlockLocations = make([]PieceBlockLocation, extra) + } + + for i := 0; i < int(extra); i++ { + + var v PieceBlockLocation + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.PieceBlockLocations[i] = v + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } } return nil diff --git a/retrievalmarket/storage_retrieval_integration_test.go b/retrievalmarket/storage_retrieval_integration_test.go index 539bc726..e14a6028 100644 --- a/retrievalmarket/storage_retrieval_integration_test.go +++ b/retrievalmarket/storage_retrieval_integration_test.go @@ -30,6 +30,7 @@ import ( "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/piecestore" + piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" @@ -244,7 +245,19 @@ func newStorageHarness(ctx context.Context, t *testing.T) *storageHarness { tempPath, err := ioutil.TempDir("", "storagemarket_test") require.NoError(t, err) - ps := piecestore.NewPieceStore(td.Ds2) + ps, err := piecestoreimpl.NewPieceStore(td.Ds2) + require.NoError(t, err) + ready := make(chan struct{}) + ps.OnReady(func() { + close(ready) + }) + err = ps.Start(ctx) + assert.NoError(t, err) + select { + case <-ready: + case <-ctx.Done(): + t.Error("did not complete migrations for piecestore") + } providerNode := &testnodes.FakeProviderNode{ FakeCommonNode: testnodes.FakeCommonNode{ SMState: smState, diff --git a/shared_testutil/test_piecestore.go b/shared_testutil/test_piecestore.go index 25eb6624..e619b59f 100644 --- a/shared_testutil/test_piecestore.go +++ b/shared_testutil/test_piecestore.go @@ -1,6 +1,7 @@ package shared_testutil import ( + "context" "errors" "testing" @@ -144,3 +145,10 @@ func (tps *TestPieceStore) ListCidInfoKeys() ([]cid.Cid, error) { func (tps *TestPieceStore) ListPieceInfoKeys() ([]cid.Cid, error) { panic("do not call me") } + +func (tps *TestPieceStore) Start(ctx context.Context) error { + return nil +} + +func (tps *TestPieceStore) OnReady(ready piecestore.ReadyFunc) { +} diff --git a/storagemarket/integration_test.go b/storagemarket/integration_test.go index 634d34c9..eb75bead 100644 --- a/storagemarket/integration_test.go +++ b/storagemarket/integration_test.go @@ -30,7 +30,7 @@ import ( "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" - "github.com/filecoin-project/go-fil-markets/piecestore" + piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/shared_testutil" @@ -462,7 +462,19 @@ func newHarnessWithTestData(t *testing.T, ctx context.Context, td *shared_testut assert.NoError(t, err) } - ps := piecestore.NewPieceStore(td.Ds2) + ps, err := piecestoreimpl.NewPieceStore(td.Ds2) + require.NoError(t, err) + ready := make(chan struct{}) + ps.OnReady(func() { + close(ready) + }) + err = ps.Start(ctx) + assert.NoError(t, err) + select { + case <-ready: + case <-ctx.Done(): + t.Error("did not complete migrations for piecestore") + } providerNode := &testnodes.FakeProviderNode{ FakeCommonNode: testnodes.FakeCommonNode{ DelayFakeCommonNode: delayFakeEnvNode, From b385e6a36acfee50fcea66b43abb833e04cb6ff2 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 25 Sep 2020 02:07:48 -0700 Subject: [PATCH 2/3] style(imports): fix imports --- piecestore/impl/piecestore.go | 1 + piecestore/migrations/migrations.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/piecestore/impl/piecestore.go b/piecestore/impl/piecestore.go index 76a7a266..1ae58bcc 100644 --- a/piecestore/impl/piecestore.go +++ b/piecestore/impl/piecestore.go @@ -12,6 +12,7 @@ import ( versioning "github.com/filecoin-project/go-ds-versioning/pkg" versioned "github.com/filecoin-project/go-ds-versioning/pkg/statestore" + "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/piecestore/migrations" ) diff --git a/piecestore/migrations/migrations.go b/piecestore/migrations/migrations.go index 7399f7eb..f80aa02c 100644 --- a/piecestore/migrations/migrations.go +++ b/piecestore/migrations/migrations.go @@ -1,11 +1,13 @@ package migrations import ( + "github.com/ipfs/go-cid" + versioning "github.com/filecoin-project/go-ds-versioning/pkg" "github.com/filecoin-project/go-ds-versioning/pkg/versioned" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-fil-markets/piecestore" ) //go:generate cbor-gen-for PieceInfo0 DealInfo0 BlockLocation0 PieceBlockLocation0 CIDInfo0 From 6ceb959a59ed7d573fbc3e3b81e0a00592bb9b2f Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 25 Sep 2020 02:12:28 -0700 Subject: [PATCH 3/3] style(lint): fix lint errors --- piecestore/impl/piecestore.go | 5 ++++- piecestore/impl/piecestore_test.go | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/piecestore/impl/piecestore.go b/piecestore/impl/piecestore.go index 1ae58bcc..27c211ad 100644 --- a/piecestore/impl/piecestore.go +++ b/piecestore/impl/piecestore.go @@ -73,7 +73,10 @@ func (ps *pieceStore) Start(ctx context.Context) error { if err != nil { log.Errorf("Migrating cidInfos: %s", err.Error()) } - ps.readySub.Publish(struct{}{}) + err = ps.readySub.Publish(struct{}{}) + if err != nil { + log.Warnf("Publish piecestore migration ready event: %s", err.Error()) + } }() return nil } diff --git a/piecestore/impl/piecestore_test.go b/piecestore/impl/piecestore_test.go index 2512405f..11966fb1 100644 --- a/piecestore/impl/piecestore_test.go +++ b/piecestore/impl/piecestore_test.go @@ -138,17 +138,17 @@ func TestStoreCIDInfo(t *testing.T) { ci, err := ps.GetCIDInfo(testCIDs[0]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[0], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[0], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[1]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[1], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[1], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[2]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[2], PieceCID: pieceCid1}) }) t.Run("overlapping adds", func(t *testing.T) { @@ -169,18 +169,18 @@ func TestStoreCIDInfo(t *testing.T) { ci, err := ps.GetCIDInfo(testCIDs[0]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[0], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[0], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[1]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 2) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid1}) - assert.Equal(t, ci.PieceBlockLocations[1], piecestore.PieceBlockLocation{blockLocations[1], pieceCid2}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[2], PieceCID: pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[1], piecestore.PieceBlockLocation{BlockLocation: blockLocations[1], PieceCID: pieceCid2}) ci, err = ps.GetCIDInfo(testCIDs[2]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid2}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[2], PieceCID: pieceCid2}) }) t.Run("duplicate adds", func(t *testing.T) { @@ -201,17 +201,17 @@ func TestStoreCIDInfo(t *testing.T) { ci, err := ps.GetCIDInfo(testCIDs[0]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[0], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[0], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[1]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[1], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[1], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[2]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[2], PieceCID: pieceCid1}) }) } @@ -317,16 +317,16 @@ func TestMigrations(t *testing.T) { ci, err := ps.GetCIDInfo(testCIDs[0]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[0], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[0], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[1]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[1], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[1], PieceCID: pieceCid1}) ci, err = ps.GetCIDInfo(testCIDs[2]) assert.NoError(t, err) assert.Len(t, ci.PieceBlockLocations, 1) - assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{blockLocations[2], pieceCid1}) + assert.Equal(t, ci.PieceBlockLocations[0], piecestore.PieceBlockLocation{BlockLocation: blockLocations[2], PieceCID: pieceCid1}) }) }